Merge upstream master

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:10
+        image: postgres:11
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres

.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+---
+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+        language_version: python3.8
+        args:
+          - "--target-version"
+          - "py37"
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changes
 
+#### Fixes
+
+- Fix inconsistent changes with JSONField ([#355](https://github.com/jazzband/django-auditlog/pull/355))
+
+## 2.0.0 (2022-05-09)
+
+#### Improvements
+- feat: enable use of replica database (delegating the choice to `DATABASES_ROUTER`) ([#359](https://github.com/jazzband/django-auditlog/pull/359))
+- Add `mask_fields` argument in `register` to mask sensitive information when logging ([#310](https://github.com/jazzband/django-auditlog/pull/310))
+- Django: Drop 2.2 support. `django_jsonfield_backport` is not required anymore ([#370](https://github.com/jazzband/django-auditlog/pull/370))
+- Remove `default_app_config` configuration ([#372](https://github.com/jazzband/django-auditlog/pull/372))
+
+#### Important notes
+- LogEntry no longer save to same database instance is using
+
 ## 1.0.0 (2022-01-24)
 
 ### Final

README.md

@@ -8,9 +8,6 @@ django-auditlog
 [![Supported Python versions](https://img.shields.io/pypi/pyversions/django-auditlog.svg)](https://pypi.python.org/pypi/django-auditlog)
 [![Supported Django versions](https://img.shields.io/pypi/djversions/django-auditlog.svg)](https://pypi.python.org/pypi/django-auditlog)
 
-**Please remember that this app is still in development.**
-**Test this app before deploying it in production environments.**
-
 ```django-auditlog``` (Auditlog) is a reusable app for Django that makes logging object changes a breeze. Auditlog tries to use as much as Python and Django's built in functionality to keep the list of dependencies as short as possible. Also, Auditlog aims to be fast and simple to use.
 
 Auditlog is created out of the need for a simple Django app that logs changes to models along with the user who made the changes (later referred to as actor). Existing solutions seemed to offer a type of version control, which was found excessive and expensive in terms of database storage and performance.

auditlog/__init__.py

@@ -6,6 +6,3 @@ try:
 except DistributionNotFound:
     # package is not installed
     pass
-
-if django.VERSION < (3, 2):
-    default_app_config = "auditlog.apps.AuditlogConfig"

auditlog/diff.py

@@ -1,6 +1,6 @@
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
-from django.db.models import NOT_PROVIDED, DateTimeField, Model
+from django.db.models import NOT_PROVIDED, DateTimeField, JSONField, Model
 from django.utils import timezone
 from django.utils.encoding import smart_str
 
@@ -58,24 +58,36 @@ def get_field_value(obj, field):
     :return: The value of the field as a string.
     :rtype: str
     """
-    if isinstance(field, DateTimeField):
-        # DateTimeFields are timezone-aware, so we need to convert the field
-        # to its naive form before we can accurately compare them for changes.
-        try:
+    try:
+        if isinstance(field, DateTimeField):
+            # DateTimeFields are timezone-aware, so we need to convert the field
+            # to its naive form before we can accurately compare them for changes.
             value = field.to_python(getattr(obj, field.name, None))
             if value is not None and settings.USE_TZ and not timezone.is_naive(value):
                 value = timezone.make_naive(value, timezone=timezone.utc)
-        except ObjectDoesNotExist:
-            value = field.default if field.default is not NOT_PROVIDED else None
-    else:
-        try:
+        elif isinstance(field, JSONField):
+            value = field.to_python(getattr(obj, field.name, None))
+        else:
             value = smart_str(getattr(obj, field.name, None))
-        except ObjectDoesNotExist:
-            value = field.default if field.default is not NOT_PROVIDED else None
+    except ObjectDoesNotExist:
+        value = field.default if field.default is not NOT_PROVIDED else None
 
     return value
 
 
+def mask_str(value: str) -> str:
+    """
+    Masks the first half of the input string to remove sensitive data.
+
+    :param value: The value to mask.
+    :type value: str
+    :return: The masked version of the string.
+    :rtype: str
+    """
+    mask_limit = int(len(value) / 2)
+    return "*" * mask_limit + value[mask_limit:]
+
+
 def model_instance_diff(old, new, fields_to_check=None):
     """
     Calculates the differences between two model instances. One of the instances may be ``None`` (i.e., a newly
@@ -145,7 +157,13 @@ def model_instance_diff(old, new, fields_to_check=None):
         new_value = get_field_value(new, field)
 
         if old_value != new_value:
-            diff[field.name] = (smart_str(old_value), smart_str(new_value))
+            if model_fields and field.name in model_fields["mask_fields"]:
+                diff[field.name] = (
+                    mask_str(smart_str(old_value)),
+                    mask_str(smart_str(new_value)),
+                )
+            else:
+                diff[field.name] = (smart_str(old_value), smart_str(new_value))
 
     if len(diff) == 0:
         diff = None

auditlog/migrations/0004_logentry_detailed_object_repr.py

@@ -1,5 +1,4 @@
-from django.db import migrations
-from django_jsonfield_backport.models import JSONField
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -12,6 +11,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name="logentry",
             name="additional_data",
-            field=JSONField(null=True, blank=True),
+            field=models.JSONField(null=True, blank=True),
         ),
     ]

auditlog/migrations/0005_logentry_additional_data_verbose_name.py

@@ -1,5 +1,4 @@
-from django.db import migrations
-from django_jsonfield_backport.models import JSONField
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -12,6 +11,8 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name="logentry",
             name="additional_data",
-            field=JSONField(null=True, verbose_name="additional data", blank=True),
+            field=models.JSONField(
+                null=True, verbose_name="additional data", blank=True
+            ),
         ),
     ]

auditlog/migrations/0011_alter_logentry_additional_data.py

@@ -1,5 +1,4 @@
-from django.db import migrations
-from django_jsonfield_backport.models import JSONField
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -12,6 +11,8 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name="logentry",
             name="additional_data",
-            field=JSONField(blank=True, null=True, verbose_name="additional data"),
+            field=models.JSONField(
+                blank=True, null=True, verbose_name="additional data"
+            ),
         ),
     ]

auditlog/models.py

@@ -12,7 +12,6 @@ from django.db.models import Q, QuerySet
 from django.utils import formats, timezone
 from django.utils.encoding import smart_str
 from django.utils.translation import gettext_lazy as _
-from django_jsonfield_backport.models import JSONField
 
 
 class LogEntryManager(models.Manager):
@@ -67,13 +66,7 @@ class LogEntryManager(models.Manager):
                         content_type=kwargs.get("content_type"),
                         object_pk=kwargs.get("object_pk", ""),
                     ).delete()
-            # save LogEntry to same database instance is using
-            db = instance._state.db
-            return (
-                self.create(**kwargs)
-                if db is None or db == ""
-                else self.using(db).create(**kwargs)
-            )
+            return self.create(**kwargs)
         return None
 
     def log_m2m_changes(
@@ -277,7 +270,7 @@ class LogEntry(models.Model):
         blank=True, null=True, verbose_name=_("remote address")
     )
     timestamp = models.DateTimeField(auto_now_add=True, verbose_name=_("timestamp"))
-    additional_data = JSONField(
+    additional_data = models.JSONField(
         blank=True, null=True, verbose_name=_("additional data")
     )
 

auditlog/registry.py

@@ -50,6 +50,7 @@ class AuditlogModelRegistry:
         include_fields: Optional[List[str]] = None,
         exclude_fields: Optional[List[str]] = None,
         mapping_fields: Optional[Dict[str, str]] = None,
+        mask_fields: Optional[List[str]] = None,
         m2m_fields: Optional[Collection[str]] = None,
     ):
         """
@@ -59,6 +60,7 @@ class AuditlogModelRegistry:
         :param include_fields: The fields to include. Implicitly excludes all other fields.
         :param exclude_fields: The fields to exclude. Overrides the fields to include.
         :param mapping_fields: Mapping from field names to strings in diff.
+        :param mask_fields: The fields to mask for sensitive info.
         :param m2m_fields: The fields to map as many to many.
 
         """
@@ -69,6 +71,8 @@ class AuditlogModelRegistry:
             exclude_fields = []
         if mapping_fields is None:
             mapping_fields = {}
+        if mask_fields is None:
+            mask_fields = []
         if m2m_fields is None:
             m2m_fields = set()
 
@@ -81,6 +85,7 @@ class AuditlogModelRegistry:
                 "include_fields": include_fields,
                 "exclude_fields": exclude_fields,
                 "mapping_fields": mapping_fields,
+                "mask_fields": mask_fields,
                 "m2m_fields": m2m_fields,
             }
             self._connect_signals(cls)
@@ -130,6 +135,7 @@ class AuditlogModelRegistry:
             "include_fields": list(self._registry[model]["include_fields"]),
             "exclude_fields": list(self._registry[model]["exclude_fields"]),
             "mapping_fields": dict(self._registry[model]["mapping_fields"]),
+            "mask_fields": list(self._registry[model]["mask_fields"]),
         }
 
     def _connect_signals(self, model):

auditlog_tests/__init__.py

@@ -1,4 +0,0 @@
-import django
-
-if django.VERSION < (3, 2):
-    default_app_config = "auditlog_tests.apps.AuditlogTestConfig"

auditlog_tests/models.py

@@ -135,6 +135,18 @@ class SimpleMappingModel(models.Model):
     history = AuditlogHistoryField()
 
 
+@auditlog.register(mask_fields=["address"])
+class SimpleMaskedModel(models.Model):
+    """
+    A simple model used for register's mask_fields kwarg
+    """
+
+    address = models.CharField(max_length=100)
+    text = models.TextField()
+
+    history = AuditlogHistoryField()
+
+
 class AdditionalDataIncludedModel(models.Model):
     """
     A model where get_additional_data is defined which allows for logging extra
@@ -238,6 +250,12 @@ class NoDeleteHistoryModel(models.Model):
     history = AuditlogHistoryField(delete_related=False)
 
 
+class JSONModel(models.Model):
+    json = models.JSONField(default=dict)
+
+    history = AuditlogHistoryField(delete_related=False)
+
+
 auditlog.register(AltPrimaryKeyModel)
 auditlog.register(UUIDPrimaryKeyModel)
 auditlog.register(ProxyModel)
@@ -255,3 +273,4 @@ auditlog.register(ChoicesFieldModel)
 auditlog.register(CharfieldTextfieldModel)
 auditlog.register(PostgresArrayFieldModel)
 auditlog.register(NoDeleteHistoryModel)
+auditlog.register(JSONModel)

auditlog_tests/test_settings.py

@@ -14,7 +14,6 @@ INSTALLED_APPS = [
     "django.contrib.sessions",
     "django.contrib.admin",
     "django.contrib.staticfiles",
-    "django_jsonfield_backport",
     "auditlog",
     "auditlog_tests",
 ]

auditlog_tests/tests.py

@@ -1,5 +1,6 @@
 import datetime
 import itertools
+import json
 from unittest import mock
 
 from dateutil.tz import gettz
@@ -12,6 +13,7 @@ from django.test import RequestFactory, TestCase
 from django.utils import dateformat, formats, timezone
 
 from auditlog.context import set_actor
+from auditlog.diff import model_instance_diff
 from auditlog.middleware import AuditlogMiddleware
 from auditlog.models import LogEntry
 from auditlog.registry import auditlog
@@ -22,6 +24,7 @@ from auditlog_tests.models import (
     ChoicesFieldModel,
     DateTimeFieldModel,
     FirstManyRelatedModel,
+    JSONModel,
     ManyRelatedModel,
     NoDeleteHistoryModel,
     OtherManyRelatedModel,
@@ -31,6 +34,7 @@ from auditlog_tests.models import (
     SimpleExcludeModel,
     SimpleIncludeModel,
     SimpleMappingModel,
+    SimpleMaskedModel,
     SimpleModel,
     UUIDPrimaryKeyModel,
 )
@@ -116,8 +120,9 @@ class SimpleModelTest(TestCase):
         obj.boolean = True
         obj.save(update_fields=[])
 
-        self.assertTrue(
-            obj.history.filter(action=LogEntry.Action.UPDATE).count() == 0,
+        self.assertEqual(
+            obj.history.filter(action=LogEntry.Action.UPDATE).count(),
+            0,
             msg="There is no log entries created",
         )
         obj.refresh_from_db()
@@ -161,6 +166,24 @@ class SimpleModelTest(TestCase):
         self.setUp()
         self.test_create()
 
+    def test_create_log_to_object_from_other_database(self):
+        msg = "The log should not try to write to the same database as the object"
+
+        instance = self.obj
+        # simulate object obtained from a different database (read only)
+        instance._state.db = "replica"
+
+        changes = model_instance_diff(None, instance)
+
+        log_entry = LogEntry.objects.log_create(
+            instance,
+            action=LogEntry.Action.CREATE,
+            changes=json.dumps(changes),
+        )
+        self.assertEqual(
+            log_entry._state.db, "default", msg=msg
+        )  # must be created in default database
+
 
 class NoActorMixin:
     def check_create_log_entry(self, obj, log_entry):
@@ -426,8 +449,9 @@ class SimpleIncludeModelTest(TestCase):
         obj.text = "New text"
         obj.save(update_fields=["text"])
 
-        self.assertTrue(
-            obj.history.filter(action=LogEntry.Action.UPDATE).count() == 0,
+        self.assertEqual(
+            obj.history.filter(action=LogEntry.Action.UPDATE).count(),
+            0,
             msg="Text change was not logged, even when passed explicitly",
         )
 
@@ -465,8 +489,9 @@ class SimpleExcludeModelTest(TestCase):
         obj.text = "New text"
         obj.save(update_fields=["text"])
 
-        self.assertTrue(
-            obj.history.filter(action=LogEntry.Action.UPDATE).count() == 0,
+        self.assertEqual(
+            obj.history.filter(action=LogEntry.Action.UPDATE).count(),
+            0,
             msg="Text change was not logged, even when passed explicitly",
         )
 
@@ -527,6 +552,19 @@ class SimpleMappingModelTest(TestCase):
         )
 
 
+class SimpeMaskedFieldsModelTest(TestCase):
+    """Log masked changes for fields in mask_fields"""
+
+    def test_register_mask_fields(self):
+        smm = SimpleMaskedModel(address="Sensitive data", text="Looong text")
+        smm.save()
+        self.assertEqual(
+            smm.history.latest().changes_dict["address"][1],
+            "*******ve data",
+            msg="The diff function masks 'address' field.",
+        )
+
+
 class AdditionalDataModelTest(TestCase):
     """Log additional data if get_additional_data is defined in the model"""
 
@@ -944,7 +982,7 @@ class CharfieldTextfieldModelTest(TestCase):
     def test_changes_display_dict_longchar(self):
         self.assertEqual(
             self.obj.history.latest().changes_display_dict["longchar"][1],
-            "{}...".format(self.PLACEHOLDER_LONGCHAR[:140]),
+            f"{self.PLACEHOLDER_LONGCHAR[:140]}...",
             msg="The string should be truncated at 140 characters with an ellipsis at the end.",
         )
         SHORTENED_PLACEHOLDER = self.PLACEHOLDER_LONGCHAR[:139]
@@ -959,7 +997,7 @@ class CharfieldTextfieldModelTest(TestCase):
     def test_changes_display_dict_longtextfield(self):
         self.assertEqual(
             self.obj.history.latest().changes_display_dict["longtextfield"][1],
-            "{}...".format(self.PLACEHOLDER_LONGTEXTFIELD[:140]),
+            f"{self.PLACEHOLDER_LONGTEXTFIELD[:140]}...",
             msg="The string should be truncated at 140 characters with an ellipsis at the end.",
         )
         SHORTENED_PLACEHOLDER = self.PLACEHOLDER_LONGTEXTFIELD[:139]
@@ -1000,7 +1038,9 @@ class PostgresArrayFieldModelTest(TestCase):
         self.obj.arrayfield = []
         self.obj.save()
         self.assertEqual(
-            self.latest_array_change, "", msg="The human readable text '' is displayed."
+            self.latest_array_change,
+            "",
+            msg="The human readable text '' is displayed.",
         )
         self.obj.arrayfield = [PostgresArrayFieldModel.GREEN]
         self.obj.save()
@@ -1068,3 +1108,90 @@ class NoDeleteHistoryTest(TestCase):
             list(entries.values_list("action", flat=True)),
             [LogEntry.Action.CREATE, LogEntry.Action.UPDATE, LogEntry.Action.DELETE],
         )
+
+
+class JSONModelTest(TestCase):
+    def setUp(self):
+        self.obj = JSONModel.objects.create()
+
+    def test_update(self):
+        """Changes on a JSONField are logged correctly."""
+        # Get the object to work with
+        obj = self.obj
+
+        # Change something
+        obj.json = {
+            "quantity": "1",
+        }
+        obj.save()
+
+        # Check for log entries
+        self.assertEqual(
+            obj.history.filter(action=LogEntry.Action.UPDATE).count(),
+            1,
+            msg="There is one log entry for 'UPDATE'",
+        )
+
+        history = obj.history.get(action=LogEntry.Action.UPDATE)
+
+        self.assertJSONEqual(
+            history.changes,
+            '{"json": ["{}", "{\'quantity\': \'1\'}"]}',
+            msg="The change is correctly logged",
+        )
+
+    def test_update_with_no_changes(self):
+        """No changes are logged."""
+        first_json = {
+            "quantity": "1814",
+            "tax_rate": "17",
+            "unit_price": "144",
+            "description": "Method form.",
+            "discount_rate": "42",
+            "unit_of_measure": "bytes",
+        }
+        obj = JSONModel.objects.create(json=first_json)
+
+        # Change the order of the keys but not the values
+        second_json = {
+            "tax_rate": "17",
+            "description": "Method form.",
+            "quantity": "1814",
+            "unit_of_measure": "bytes",
+            "unit_price": "144",
+            "discount_rate": "42",
+        }
+        obj.json = second_json
+        obj.save()
+
+        # Check for log entries
+        self.assertEqual(
+            first_json,
+            second_json,
+            msg="dicts are the same",
+        )
+        self.assertEqual(
+            obj.history.filter(action=LogEntry.Action.UPDATE).count(),
+            0,
+            msg="There is no log entry",
+        )
+
+
+class ModelInstanceDiffTest(TestCase):
+    def test_when_field_doesnt_exit(self):
+        """No error is raised and the default is returned."""
+        first = SimpleModel(boolean=True)
+        second = SimpleModel()
+
+        # then boolean should be False, as we use the default value
+        # specified inside the model
+        del second.boolean
+
+        changes = model_instance_diff(first, second)
+
+        # Check for log entries
+        self.assertEqual(
+            changes,
+            {"boolean": ("True", "False")},
+            msg="ObjectDoesNotExist should be handled",
+        )

docs/requirements.txt

@@ -3,4 +3,3 @@ django>=3.2,<3.3
 sphinx
 sphinx_rtd_theme
 psycopg2-binary
-django-jsonfield-backport>=1.0.0

docs/source/installation.rst

@@ -12,9 +12,9 @@ The repository can be found at https://github.com/jazzband/django-auditlog/.
 **Requirements**
 
 - Python 3.7 or higher
-- Django 2.2 or higher
+- Django 3.2 or higher
 
-Auditlog is currently tested with Python 3.7+ and Django 2.2, 3.2 and 4.0. The latest test report can be found
+Auditlog is currently tested with Python 3.7+ and Django 3.2 and 4.0. The latest test report can be found
 at https://github.com/jazzband/django-auditlog/actions.
 
 Adding Auditlog to your Django application

docs/source/usage.rst

@@ -61,7 +61,7 @@ during the `register()` call.
         sku = models.CharField(max_length=20)
         version = models.CharField(max_length=5)
         product = models.CharField(max_length=50, verbose_name='Product Name')
-        history = AuditLogHistoryField()
+        history = AuditlogHistoryField()
         
     auditlog.register(MyModel, mapping_fields={'sku': 'Product No.', 'version': 'Product Revision'})
     
@@ -76,6 +76,21 @@ during the `register()` call.
 
 You do not need to map all the fields of the model, any fields not mapped will fall back on their ``verbose_name``. Django provides a default ``verbose_name`` which is a "munged camel case version" so ``product_name`` would become ``Product Name`` by default.
 
+**Masking fields**
+
+Fields that contain sensitive info and we want keep track of field change but not to contain the exact change.
+
+To mask specific fields from the log you can pass ``mask_fields`` to the ``register``
+method. If ``mask_fields`` is specified, the first half value of the fields is masked using ``*``.
+
+For example, to mask the field ``address``, use::
+
+    auditlog.register(MyModel, mask_fields=['address'])
+
+.. versionadded:: 2.0.0
+
+    Masking fields
+
 Actors
 ------
 

setup.py

@@ -30,7 +30,6 @@ setup(
     long_description_content_type="text/markdown",
     install_requires=[
         "django-admin-rangefilter>=0.8.0",
-        "django-jsonfield-backport>=1.0.0",
         "python-dateutil>=2.6.0",
     ],
     zip_safe=False,
@@ -41,7 +40,6 @@ setup(
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Framework :: Django",
-        "Framework :: Django :: 2.2",
         "Framework :: Django :: 3.2",
         "Framework :: Django :: 4.0",
         "License :: OSI Approved :: MIT License",

tox.ini

@@ -1,10 +1,9 @@
 [tox]
 envlist =
-    {py37,py38,py39}-django22
     {py37,py38,py39,py310}-django32
     {py38,py39,py310}-django{40,main}
     py38-docs
-    py38-qa
+    py38-lint
 
 [testenv]
 setenv =
@@ -13,7 +12,6 @@ commands =
     coverage run --source auditlog runtests.py
     coverage xml
 deps =
-    django22: Django>=2.2,<2.3
     django32: Django>=3.2,<3.3
     django40: Django>=4.0,<4.1
     djangomain: https://github.com/django/django/archive/main.tar.gz
@@ -40,14 +38,10 @@ changedir = docs/source
 deps = -rdocs/requirements.txt
 commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html
 
-[testenv:py38-qa]
-basepython = python3.8
-deps =
-    black
-    isort
+[testenv:py38-lint]
+deps = pre-commit
 commands =
-    black --check --diff auditlog auditlog_tests setup.py runtests.py
-    isort --check-only --diff auditlog auditlog_tests setup.py runtests.py
+    pre-commit run --all-files
 
 [gh-actions]
 python =