From d2fc375b985897ae138deb6fcc1935a50410b135 Mon Sep 17 00:00:00 2001 From: Drew Hubl Date: Fri, 29 Jul 2016 14:58:04 -0600 Subject: [PATCH] Use cache versioning instead of invalidating the entire cache --- README.rst | 287 +++++++++++++++++++++++++++------------------- tos/apps.py | 14 ++- tos/middleware.py | 20 +++- 3 files changed, 198 insertions(+), 123 deletions(-) diff --git a/README.rst b/README.rst index 7623c34..6c38fe9 100644 --- a/README.rst +++ b/README.rst @@ -11,18 +11,18 @@ This project gives the admin the ability to reset terms of agreement with the en Summary ======= - - Keeps track of when TOS is changed - - Users need to be informed and agree/re-agree when they login (custom login is provided) - - Just two models (TOS and user agreement) +- Keeps track of when TOS is changed +- Users need to be informed and agree/re-agree when they login (custom login is provided) +- Just two models (TOS and user agreement) Terms Of Service Installation ============================= - 1. `pip install django-tos` +1. ``pip install django-tos`` - 2. Add `tos` to your `INSTALLED_APPS` setting. +2. Add ``tos`` to your ``INSTALLED_APPS`` setting. - 3. Sync your database with `python manage.py migrate` or `python manage.py syncdb` for Django < 1.7. +3. Sync your database with ``python manage.py migrate`` or ``python manage.py syncdb`` for Django < 1.7. Configuration ============= @@ -30,34 +30,16 @@ Configuration Options ``````` -There are two ways to configure `django-tos` - either enable the TOS check when users sign in, or use middleware to enable the TOS check on every `GET` request. +There are two ways to configure ``django-tos`` - either enable the TOS check when users sign in, or use middleware to enable the TOS check on every ``GET`` request. If you cannot override your login view (for instance, if you're using `django-allauth `_) you should use the second option. -There are some additional advantages of using the middleware option: +Option 1: TOS Check On Sign In +`````````````````````````````` -* Allow some of your users to skip the TOS check (eg: developers, staff, admin, superusers, employees) -* Best case for staff users: 1 cache hit -* Best case for non-staff users: 1 cache miss, 1 cache hit -* Uses signals to invalidate cached agreements -* Skips the agreement check when the user is anonymous or not signed in -* Skips the agreement check when the request is AJAX -* Skips the agreement check when the request isn't a `GET` request (to avoid getting in the way of data mutations) - -But there are some disadvantages: +In your root urlconf file ``urls.py`` add: -* Worst case: 1 cache miss, 1 database query, 1 cache set (this should only happen when the user signs in) -* Requires a separate cache for `django-tos` because... -* Invalidates **entire** TOS cache when `TermsOfService` is changed -* Requires a cache key for each user who is signed in -* Requires an additional cache key for each staff user - -Because of this, the middleware is 100% optional. - -TOS Check On Login -`````````````````` - -In your root urlconf file `urls.py` add:: +.. code-block:: python # terms of service links urlpatterns += patterns('', @@ -65,89 +47,148 @@ In your root urlconf file `urls.py` add:: url(r'^terms-of-service/', include('tos.urls')), ) -Middleware Check -```````````````` +Option 2: Middleware Check +`````````````````````````` -1. In your root urlconf file `urls.py` only add the terms-of-service URLs:: +This option uses the ``incr`` methods for the configured Django cache. If you are using ``django-tos`` in a complex or parallel environment, be sure to use a cache backend that supports atomic increment operations. For more information, see the notes at the end of `this section of the Django documentation `_. - # terms of service links - urlpatterns += patterns('', - url(r'^terms-of-service/', include('tos.urls')), - ) +Also, to ensure that warming the cache with users who can skip the agreement check works properly, you will need to include ``tos`` before your app (``myapp`` in the example) in your ``INSTALLED_APPS`` setting: -2. It is strongly recommended to use a cache specifically for `django-tos`, because it clears ALL keys in the configured cache when `TermsOfService` objects change. Create a new cache in your project's `settings.py`:: - - CACHES = { - ... - # The cache to use specifically for django-tos - 'tos': { # Can use any name - 'BACKEND': ..., - 'LOCATION': ..., - 'NAME': 'tos-cache', # Can use any name - }, - } +.. code-block:: python -3. Configure `django-tos` to use the cache:: - - TOS_CACHE_NAME = 'tos' # Much match the key name in in `CACHES`. - -4. Then in your project's `settings.py` add the middleware to `MIDDLEWARE_CLASSES`:: - - MIDDLEWARE_CLASSES = ( + INSTALLED_APPS = ( + ... + 'tos', + ... + 'myapp', # Example app name ... - # Terms of service checks - 'tos.middleware.UserAgreementMiddleware', ) -5. To allow users to skip the TOS check, you will need to set corresponding cache keys for them in the TOS cache. The cache key for each user will need to be prefixed with 'django:tos:skip_tos_check:', and have the user ID appended to it. +Advantages +---------- - Here is an example app configuration that allows staff users and superusers to skip the tos check: +* Can optionally use a separate cache for TOS agreements (necessary if your default cache does not support atomic increment operations) +* Allow some of your users to skip the TOS check (eg: developers, staff, admin, superusers, employees) +* Uses signals to invalidate cached agreements +* Skips the agreement check when the user is anonymous or not signed in +* Skips the agreement check when the request is AJAX +* Skips the agreement check when the request isn't a ``GET`` request (to avoid getting in the way of data mutations) + +Disadvantages +------------- + +* Requires a cache key for each user who is signed in +* Requires an additional cache key for each staff user +* May leave keys in the cache when the active ``TermsOfService`` changes + +Efficiency +---------- + +* Best case for staff users: 2 cache hits +* Best case for non-staff users: 1 cache miss, 2 cache hits +* Worst case: 1 cache hit, 2 cache misses, 1 database query, 1 cache set (this should only happen when the user signs in) + +Option 2 Configuration +---------------------- + +1. In your root urlconf file ``urls.py`` only add the terms-of-service URLs: .. code-block:: python - from django.apps import AppConfig, apps - from django.conf import settings - from django.contrib.auth import get_user_model - from django.core.cache import caches - from django.db.models import Q - from django.db.models.signals import post_save, pre_save - from django.dispatch import receiver + # terms of service links + urlpatterns += patterns('', + url(r'^terms-of-service/', include('tos.urls')), + ) - class MyAppConfig(AppConfig): - def ready(self): - if 'tos' in settings.INSTALLED_APPS: - cache = caches[getattr(settings, 'TOS_CACHE_NAME')] - tos_app = apps.get_app_config('tos') - TermsOfService = tos_app.get_model('TermsOfService') +2. Optional: Since the cache used by TOS will be overwhelmingly read-heavy, you can use a separate cache specifically for TOS. To do so, create a new cache in your project's ``settings.py``: - @receiver(post_save, sender=get_user_model(), dispatch_uid='set_staff_in_cache_for_tos') - def set_staff_in_cache_for_tos(user, instance, **kwargs): - if kwargs.get('raw', False): - return + .. code-block:: python + + CACHES = { + ... + # The cache specifically for django-tos + 'tos': { # Can use any name here + 'BACKEND': ..., + 'LOCATION': ..., + 'NAME': 'tos-cache', # Can use any name here + }, + } - # If the user is staff allow them to skip the TOS agreement check - if instance.is_staff or instance.is_superuser: - cache.set('django:tos:skip_tos_check:{}'.format(instance.id)) + and configure ``django-tos`` to use the new cache: - # But if they aren't make sure we invalidate them from the cache - elif cache.get('django:tos:skip_tos_check:{}'.format(instance.id), False): - cache.delete('django:tos:skip_tos_check:{}'.format(instance.id)) + .. code-block:: python - @receiver(post_save, sender=TermsOfService, dispatch_uid='add_staff_users_to_tos_cache') - def add_staff_users_to_tos_cache(*args, **kwargs): - if kwargs.get('raw', False): - return + TOS_CACHE_NAME = 'tos' # Must match the key name in in CACHES - # Efficiently cache all of the users who are allowed to skip the TOS - # agreement check - cache.set_many({ - 'django:tos:skip_tos_check:{}'.format(staff_user.id): True - for staff_user in get_user_model().objects.filter( - Q(is_staff=True) | Q(is_superuser=True)) - }) + this setting defaults to the ``default`` cache. - # Immediately add staff users to the cache - add_staff_users_to_tos_cache() +4. Then in your project's ``settings.py`` add the middleware to ``MIDDLEWARE_CLASSES``: + + .. code-block:: python + + MIDDLEWARE_CLASSES = ( + ... + # Terms of service checks + 'tos.middleware.UserAgreementMiddleware', + ) + +5. Optional: To allow users to skip the TOS check, you will need to set corresponding cache keys for them in the TOS cache. The cache key for each user will need to be prefixed with ``django:tos:skip_tos_check:``, and have the user ID appended to it. + + Here is an example app configuration that allows staff users and superusers to skip the TOS agreement check: + + .. code-block:: python + + from django.apps import AppConfig, apps + from django.conf import settings + from django.contrib.auth import get_user_model + from django.core.cache import caches + from django.db.models import Q + from django.db.models.signals import post_save, pre_save + from django.dispatch import receiver + + class MyAppConfig(AppConfig): + name = 'myapp' + + def ready(self): + if 'tos' in settings.INSTALLED_APPS: + cache = caches[getattr(settings, 'TOS_CACHE_NAME', 'default')] + tos_app = apps.get_app_config('tos') + TermsOfService = tos_app.get_model('TermsOfService') + + @receiver(post_save, sender=get_user_model(), dispatch_uid='set_staff_in_cache_for_tos') + def set_staff_in_cache_for_tos(user, instance, **kwargs): + if kwargs.get('raw', False): + return + + # Get the cache prefix + key_version = cache.get('django:tos:key_version') + + # If the user is staff allow them to skip the TOS agreement check + if instance.is_staff or instance.is_superuser: + cache.set('django:tos:skip_tos_check:{}'.format(instance.id), version=key_version) + + # But if they aren't make sure we invalidate them from the cache + elif cache.get('django:tos:skip_tos_check:{}'.format(instance.id), False): + cache.delete('django:tos:skip_tos_check:{}'.format(instance.id), version=key_version) + + @receiver(post_save, sender=TermsOfService, dispatch_uid='add_staff_users_to_tos_cache') + def add_staff_users_to_tos_cache(*args, **kwargs): + if kwargs.get('raw', False): + return + + # Get the cache prefix + key_version = cache.get('django:tos:key_version') + + # Efficiently cache all of the users who are allowed to skip the TOS + # agreement check + cache.set_many({ + 'django:tos:skip_tos_check:{}'.format(staff_user.id): True + for staff_user in get_user_model().objects.filter( + Q(is_staff=True) | Q(is_superuser=True)) + }, version=key_version) + + # Immediately add staff users to the cache + add_staff_users_to_tos_cache() =============== django-tos-i18n @@ -159,12 +200,16 @@ Terms Of Service i18n Installation ================================== Assuming you have correctly installed django-tos in your app you only need to -add following apps to ``INSTALLED_APPS``:: +add following apps to ``INSTALLED_APPS``: + +.. code-block:: python INSTALLED_APPS += ('modeltranslation', 'tos_i18n') and also you should also define your languages in Django ``LANGUAGES`` -variable, eg.:: +variable, e.g.: + +.. code-block:: python LANGUAGES = ( ('pl', 'Polski'), @@ -173,16 +218,16 @@ variable, eg.:: Please note that adding those to ``INSTALLED_APPS`` **changes** Django models. Concretely it adds for every registered ``field`` that should translated, -additional fields with name ``field_``, e.g. for given model:: +additional fields with name ``field_``, e.g. for given model: + +.. code-block:: python class MyModel(models.Model): name = models.CharField(max_length=10) There will be generated fields: ``name`` , ``name_en``, ``name_pl``. -You should probably migrate your database, using South is recommended. -Migrations should be kept in your local project. - +You should probably migrate your database, and if you're using Django < 1.7 using South is recommended. These migrations should be kept in your local project. How to migrate tos with South ````````````````````````````` @@ -192,32 +237,44 @@ instalation synced using syncdb into a translated django-tos-i18n with South migrations. 1. Inform South that you want to store migrations in custom place by putting - this in your Django settings file:: + this in your Django settings file: - SOUTH_MIGRATION_MODULES = { - 'tos': 'YOUR_APP.migrations.tos', - } + .. code-block:: python -2. Add required directory (package):: + SOUTH_MIGRATION_MODULES = { + 'tos': 'YOUR_APP.migrations.tos', + } - mkdir -p YOUR_APP/migrations/tos - touch YOUR_APP/migrations/tos/__init__.py +2. Add required directory (package): -3. Create initial migration (referring to the database state as it is now):: + .. code-block:: bash - python manage.py schemamigration --initial tos + mkdir -p YOUR_APP/migrations/tos + touch YOUR_APP/migrations/tos/__init__.py -4. Fake migration (because the changes are already in the database):: +3. Create initial migration (referring to the database state as it is now): - python manage.py migrate tos --fake + .. code-block:: bash -5. Install tos_i18n (and modeltranslation) to ``INSTALLED_APPS``:: + python manage.py schemamigration --initial tos - INSTALLED_APPS += ('modeltranslation', 'tos_i18n',) +4. Fake migration (because the changes are already in the database): -6. Make sure that the Django LANGUAGES setting is properly configured. + .. code-block:: bash -7. Migrate what changed:: + python manage.py migrate tos --fake + +5. Install tos_i18n (and modeltranslation) to ``INSTALLED_APPS``: + + .. code-block:: python + + INSTALLED_APPS += ('modeltranslation', 'tos_i18n',) + +6. Make sure that the Django ``LANGUAGES`` setting is properly configured. + +7. Migrate what changed: + + .. code-block:: bash $ python manage.py schemamigration --auto tos $ python migrate tos diff --git a/tos/apps.py b/tos/apps.py index ac33c93..98d7491 100644 --- a/tos/apps.py +++ b/tos/apps.py @@ -15,7 +15,7 @@ class TOSConfig(AppConfig): def ready(self): if 'tos.middleware.UserAgreementMiddleware' in MIDDLEWARES: # Force the user to create a separate cache - cache = caches[getattr(settings, 'TOS_CACHE_NAME')] + cache = caches[getattr(settings, 'TOS_CACHE_NAME', 'default')] TermsOfService = self.get_model('TermsOfService') @@ -24,5 +24,13 @@ class TOSConfig(AppConfig): if kwargs.get('raw', False): return - # Efficiently clear all keys from the cache - cache.clear() + # Set the key version to 0 if it doesn't exist and leave it + # alone if it does + cache.add('django:tos:key_version', 0) + + # This key will be used to version the rest of the TOS keys + # Incrementing it will effectively invalidate all previous keys + cache.incr('django:tos:key_version') + + # Create the TOS key version immediately + invalidate_cached_agreements(TermsOfService, None) diff --git a/tos/middleware.py b/tos/middleware.py index ecf3c52..aae874a 100644 --- a/tos/middleware.py +++ b/tos/middleware.py @@ -3,10 +3,12 @@ from django.contrib.auth import SESSION_KEY as session_key from django.core.cache import caches from django.core.urlresolvers import reverse from django.http import HttpResponseRedirect +from django.utils.cache import add_never_cache_headers from .models import UserAgreement -cache = caches[getattr(settings, 'TOS_CACHE_NAME')] +cache = caches[getattr(settings, 'TOS_CACHE_NAME', 'default')] +tos_check_url = reverse('tos_check_tos') class UserAgreementMiddleware(object): @@ -22,6 +24,10 @@ class UserAgreementMiddleware(object): if request.is_ajax(): return None + # Don't redirect users when they're trying to get to the confirm page + if request.path_info == tos_check_url: + return None + # If the user doesn't have a user ID, ignore them - they're anonymous if not request.session.get(session_key, None): return None @@ -32,13 +38,16 @@ class UserAgreementMiddleware(object): # ever change (usernames and email addresses can change) user_id = request.session.get(session_key) + # Get the cache prefix + key_version = cache.get('django:tos:key_version') + # Skip if the user is allowed to skip - for instance, if the user is an # admin or a staff member - if cache.get('django:tos:skip_tos_check:{}'.format(user_id), False): + if cache.get('django:tos:skip_tos_check:{}'.format(str(user_id)), False, version=key_version): return None # Ping the cache for the user agreement - user_agreed = cache.get('django:tos:agreed:{}'.format(user_id), None) + user_agreed = cache.get('django:tos:agreed:{}'.format(str(user_id)), None, version=key_version) # If the cache is missing this user if user_agreed is None: @@ -48,9 +57,10 @@ class UserAgreementMiddleware(object): terms_of_service__active=True).exists() # Set the value in the cache - cache.set('django:tos:agreed:{}'.format(user_id), user_agreed) + cache.set('django:tos:agreed:{}'.format(user_id), user_agreed, version=key_version) if not user_agreed: - return HttpResponseRedirect('tos_check_tos') + print("User didn't agree") + return add_never_cache_headers(HttpResponseRedirect(tos_check_url)) return None