From aced88edb7d50d4308412261fd8def8ac92ce77d Mon Sep 17 00:00:00 2001 From: Andrew-Chen-Wang Date: Sun, 9 Feb 2020 22:57:35 -0500 Subject: [PATCH] Updated README, new maintainer * Added Django 3.0 support to avoid conflicts when pip installing * README shows cache comparisons and usage time * Travis updated * gitignore updated from GitHub's main page --- .gitignore | 121 ++++++++++++++++++++++++++++++++++++++++++++-- .travis.yml | 96 ++++++++++++++++++++++++++++++++++++ README.rst | 43 +++++++++++++--- cachalot/apps.py | 2 +- cachalot/utils.py | 4 +- requirements.txt | 3 +- 6 files changed, 255 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index c494e75..cbb848a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,125 @@ +# Byte-compiled / optimized / DLL files __pycache__/ -*.pyc +*.py[cod] +*$py.class + +# C extensions *.so + +# Distribution / packaging +.Python build/ +develop-eggs/ dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ *.egg-info/ -.coverage -docs/_*/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ .tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal +.idea/ + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/.travis.yml b/.travis.yml index 36ea63c..7834a31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ matrix: env: TOXENV=py2.7-django1.11-sqlite3-locmem - python: 2.7 env: TOXENV=py2.7-django1.11-sqlite3-filebased + - python: 2.7 env: TOXENV=py2.7-django1.11-postgresql-redis - python: 2.7 @@ -31,6 +32,7 @@ matrix: env: TOXENV=py2.7-django1.11-postgresql-locmem - python: 2.7 env: TOXENV=py2.7-django1.11-postgresql-filebased + - python: 2.7 env: TOXENV=py2.7-django1.11-mysql-redis - python: 2.7 @@ -41,6 +43,7 @@ matrix: env: TOXENV=py2.7-django1.11-mysql-locmem - python: 2.7 env: TOXENV=py2.7-django1.11-mysql-filebased + - python: 3.4 env: TOXENV=py3.4-django1.11-sqlite3-redis - python: 3.4 @@ -51,6 +54,7 @@ matrix: env: TOXENV=py3.4-django1.11-sqlite3-locmem - python: 3.4 env: TOXENV=py3.4-django1.11-sqlite3-filebased + - python: 3.4 env: TOXENV=py3.4-django1.11-postgresql-redis - python: 3.4 @@ -61,6 +65,7 @@ matrix: env: TOXENV=py3.4-django1.11-postgresql-locmem - python: 3.4 env: TOXENV=py3.4-django1.11-postgresql-filebased + - python: 3.4 env: TOXENV=py3.4-django1.11-mysql-redis - python: 3.4 @@ -71,6 +76,7 @@ matrix: env: TOXENV=py3.4-django1.11-mysql-locmem - python: 3.4 env: TOXENV=py3.4-django1.11-mysql-filebased + - python: 3.5 env: TOXENV=py3.5-django1.11-sqlite3-redis - python: 3.5 @@ -81,6 +87,7 @@ matrix: env: TOXENV=py3.5-django1.11-sqlite3-locmem - python: 3.5 env: TOXENV=py3.5-django1.11-sqlite3-filebased + - python: 3.5 env: TOXENV=py3.5-django1.11-postgresql-redis - python: 3.5 @@ -91,6 +98,7 @@ matrix: env: TOXENV=py3.5-django1.11-postgresql-locmem - python: 3.5 env: TOXENV=py3.5-django1.11-postgresql-filebased + - python: 3.5 env: TOXENV=py3.5-django1.11-mysql-redis - python: 3.5 @@ -101,6 +109,7 @@ matrix: env: TOXENV=py3.5-django1.11-mysql-locmem - python: 3.5 env: TOXENV=py3.5-django1.11-mysql-filebased + - python: 3.6 env: TOXENV=py3.6-django1.11-sqlite3-redis - python: 3.6 @@ -111,6 +120,7 @@ matrix: env: TOXENV=py3.6-django1.11-sqlite3-locmem - python: 3.6 env: TOXENV=py3.6-django1.11-sqlite3-filebased + - python: 3.6 env: TOXENV=py3.6-django1.11-postgresql-redis - python: 3.6 @@ -121,6 +131,7 @@ matrix: env: TOXENV=py3.6-django1.11-postgresql-locmem - python: 3.6 env: TOXENV=py3.6-django1.11-postgresql-filebased + - python: 3.6 env: TOXENV=py3.6-django1.11-mysql-redis - python: 3.6 @@ -131,6 +142,7 @@ matrix: env: TOXENV=py3.6-django1.11-mysql-locmem - python: 3.6 env: TOXENV=py3.6-django1.11-mysql-filebased + - python: 3.4 env: TOXENV=py3.4-django2.0-sqlite3-redis - python: 3.4 @@ -141,6 +153,7 @@ matrix: env: TOXENV=py3.4-django2.0-sqlite3-locmem - python: 3.4 env: TOXENV=py3.4-django2.0-sqlite3-filebased + - python: 3.4 env: TOXENV=py3.4-django2.0-postgresql-redis - python: 3.4 @@ -151,6 +164,7 @@ matrix: env: TOXENV=py3.4-django2.0-postgresql-locmem - python: 3.4 env: TOXENV=py3.4-django2.0-postgresql-filebased + - python: 3.4 env: TOXENV=py3.4-django2.0-mysql-redis - python: 3.4 @@ -161,6 +175,7 @@ matrix: env: TOXENV=py3.4-django2.0-mysql-locmem - python: 3.4 env: TOXENV=py3.4-django2.0-mysql-filebased + - python: 3.5 env: TOXENV=py3.5-django2.0-sqlite3-redis - python: 3.5 @@ -171,6 +186,7 @@ matrix: env: TOXENV=py3.5-django2.0-sqlite3-locmem - python: 3.5 env: TOXENV=py3.5-django2.0-sqlite3-filebased + - python: 3.5 env: TOXENV=py3.5-django2.0-postgresql-redis - python: 3.5 @@ -181,6 +197,7 @@ matrix: env: TOXENV=py3.5-django2.0-postgresql-locmem - python: 3.5 env: TOXENV=py3.5-django2.0-postgresql-filebased + - python: 3.5 env: TOXENV=py3.5-django2.0-mysql-redis - python: 3.5 @@ -191,6 +208,7 @@ matrix: env: TOXENV=py3.5-django2.0-mysql-locmem - python: 3.5 env: TOXENV=py3.5-django2.0-mysql-filebased + - python: 3.6 env: TOXENV=py3.6-django2.0-sqlite3-redis - python: 3.6 @@ -201,6 +219,7 @@ matrix: env: TOXENV=py3.6-django2.0-sqlite3-locmem - python: 3.6 env: TOXENV=py3.6-django2.0-sqlite3-filebased + - python: 3.6 env: TOXENV=py3.6-django2.0-postgresql-redis - python: 3.6 @@ -211,6 +230,7 @@ matrix: env: TOXENV=py3.6-django2.0-postgresql-locmem - python: 3.6 env: TOXENV=py3.6-django2.0-postgresql-filebased + - python: 3.6 env: TOXENV=py3.6-django2.0-mysql-redis - python: 3.6 @@ -221,6 +241,7 @@ matrix: env: TOXENV=py3.6-django2.0-mysql-locmem - python: 3.6 env: TOXENV=py3.6-django2.0-mysql-filebased + - python: 3.5 env: TOXENV=py3.5-django2.1-sqlite3-redis - python: 3.5 @@ -231,6 +252,7 @@ matrix: env: TOXENV=py3.5-django2.1-sqlite3-locmem - python: 3.5 env: TOXENV=py3.5-django2.1-sqlite3-filebased + - python: 3.5 env: TOXENV=py3.5-django2.1-postgresql-redis - python: 3.5 @@ -241,6 +263,7 @@ matrix: env: TOXENV=py3.5-django2.1-postgresql-locmem - python: 3.5 env: TOXENV=py3.5-django2.1-postgresql-filebased + - python: 3.5 env: TOXENV=py3.5-django2.1-mysql-redis - python: 3.5 @@ -251,6 +274,7 @@ matrix: env: TOXENV=py3.5-django2.1-mysql-locmem - python: 3.5 env: TOXENV=py3.5-django2.1-mysql-filebased + - python: 3.6 env: TOXENV=py3.6-django2.1-sqlite3-redis - python: 3.6 @@ -261,6 +285,7 @@ matrix: env: TOXENV=py3.6-django2.1-sqlite3-locmem - python: 3.6 env: TOXENV=py3.6-django2.1-sqlite3-filebased + - python: 3.6 env: TOXENV=py3.6-django2.1-postgresql-redis - python: 3.6 @@ -271,6 +296,7 @@ matrix: env: TOXENV=py3.6-django2.1-postgresql-locmem - python: 3.6 env: TOXENV=py3.6-django2.1-postgresql-filebased + - python: 3.6 env: TOXENV=py3.6-django2.1-mysql-redis - python: 3.6 @@ -281,6 +307,40 @@ matrix: env: TOXENV=py3.6-django2.1-mysql-locmem - python: 3.6 env: TOXENV=py3.6-django2.1-mysql-filebased + + - python: 3.6 + env: TOXENV=py3.6-django2.2-sqlite3-redis + - python: 3.6 + env: TOXENV=py3.6-django2.2-sqlite3-memcached + - python: 3.6 + env: TOXENV=py3.6-django2.2-sqlite3-pylibmc + - python: 3.6 + env: TOXENV=py3.6-django2.2-sqlite3-locmem + - python: 3.6 + env: TOXENV=py3.6-django2.2-sqlite3-filebased + + - python: 3.6 + env: TOXENV=py3.6-django2.2-postgresql-redis + - python: 3.6 + env: TOXENV=py3.6-django2.2-postgresql-memcached + - python: 3.6 + env: TOXENV=py3.6-django2.2-postgresql-pylibmc + - python: 3.6 + env: TOXENV=py3.6-django2.2-postgresql-locmem + - python: 3.6 + env: TOXENV=py3.6-django2.2-postgresql-filebased + + - python: 3.6 + env: TOXENV=py3.6-django2.2-mysql-redis + - python: 3.6 + env: TOXENV=py3.6-django2.2-mysql-memcached + - python: 3.6 + env: TOXENV=py3.6-django2.2-mysql-pylibmc + - python: 3.6 + env: TOXENV=py3.6-django2.2-mysql-locmem + - python: 3.6 + env: TOXENV=py3.6-django2.2-mysql-filebased + - python: 3.7 env: TOXENV=py3.7-django2.1-sqlite3-redis - python: 3.7 @@ -291,6 +351,7 @@ matrix: env: TOXENV=py3.7-django2.1-sqlite3-locmem - python: 3.7 env: TOXENV=py3.7-django2.1-sqlite3-filebased + - python: 3.7 env: TOXENV=py3.7-django2.1-postgresql-redis - python: 3.7 @@ -301,6 +362,7 @@ matrix: env: TOXENV=py3.7-django2.1-postgresql-locmem - python: 3.7 env: TOXENV=py3.7-django2.1-postgresql-filebased + - python: 3.7 env: TOXENV=py3.7-django2.1-mysql-redis - python: 3.7 @@ -311,6 +373,40 @@ matrix: env: TOXENV=py3.7-django2.1-mysql-locmem - python: 3.7 env: TOXENV=py3.7-django2.1-mysql-filebased + + - python: 3.7 + env: TOXENV=py3.7-django2.2-sqlite3-redis + - python: 3.7 + env: TOXENV=py3.7-django2.2-sqlite3-memcached + - python: 3.7 + env: TOXENV=py3.7-django2.2-sqlite3-pylibmc + - python: 3.7 + env: TOXENV=py3.7-django2.2-sqlite3-locmem + - python: 3.7 + env: TOXENV=py3.7-django2.2-sqlite3-filebased + + - python: 3.7 + env: TOXENV=py3.7-django2.2-postgresql-redis + - python: 3.7 + env: TOXENV=py3.7-django2.2-postgresql-memcached + - python: 3.7 + env: TOXENV=py3.7-django2.2-postgresql-pylibmc + - python: 3.7 + env: TOXENV=py3.7-django2.2-postgresql-locmem + - python: 3.7 + env: TOXENV=py3.7-django2.2-postgresql-filebased + + - python: 3.7 + env: TOXENV=py3.7-django2.2-mysql-redis + - python: 3.7 + env: TOXENV=py3.7-django2.2-mysql-memcached + - python: 3.7 + env: TOXENV=py3.7-django2.2-mysql-pylibmc + - python: 3.7 + env: TOXENV=py3.7-django2.2-mysql-locmem + - python: 3.7 + env: TOXENV=py3.7-django2.2-mysql-filebased + allow_failures: - env: TOXENV=py3.7-django2.1-sqlite3-redis - env: TOXENV=py3.7-django2.1-sqlite3-memcached diff --git a/README.rst b/README.rst index 1cdca7f..8690256 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,12 @@ -**WARNING: This project is in need of a new maintainer.** NoriPyt and its developer Bertrand Bordage no longer use it and therefore don’t have time to develop it. +**New Maintainer**: `Andrew Chen Wang`_ is a new maintainer of this repo. Bordage is still the admin but will most likely be inactive. -Django-cachalot +Django Cachalot =============== Caches your Django ORM queries and automatically invalidates them. +Documentation: http://django-cachalot.readthedocs.io + .. image:: https://raw.github.com/noripyt/django-cachalot/master/django-cachalot.jpg ---- @@ -21,17 +23,42 @@ Caches your Django ORM queries and automatically invalidates them. .. image:: http://img.shields.io/scrutinizer/g/noripyt/django-cachalot/master.svg?style=flat-square&maxAge=3600 :target: https://scrutinizer-ci.com/g/noripyt/django-cachalot/ -.. image:: https://img.shields.io/gitter/room/django-cachalot/Lobby.svg?style=flat-square&maxAge=3600 - :target: https://gitter.im/django-cachalot/Lobby +.. image:: https://img.shields.io/badge/cachalot-Chat%20on%20Slack-green?style=flat&logo=slack + :target: https://join.slack.com/t/cachalotdjango/shared_invite/enQtOTMyNzI0NTQzOTA3LWViYmYwMWY3MmU0OTZkYmNiMjBhN2NjNjc4OWVlZDNiMjMxN2Y3YzljYmNiYTY4ZTRjOGQxZDRiMTM0NWE3NGI +Third-Party Cache Comparison +---------------------------- -Documentation -------------- +There are three main third party caches: cachalot, cache-machine, and cache-ops. Which do you use? We suggest a mix: -Available `on Read The Docs `_. +TL;DR Use cachalot for cold or accessed <50 times per minutes (Most people should stick with only cachalot since you +most likely won't need to scale to the point of needing cache-machine added to the bowl). If you're an enterprise that +already has huge statistics, then mixing cold caches for cachalot and your hot caches with cache-machine is the best +mix. +Recall, cachalot caches THE ENTIRE TABLE. That's where its inefficiency stems from: if you keep updating the records, +then the cachalot constantly invalidates the table and re-caches. Luckily caching is very efficient, it's just the cache +invalidation part that kills all our systems. Look at Note 1 below to see how Reddit deals with it. + +Cachalot is more-or-less intended for cold caches or "just-right" conditions. If you find a partition library for +Django (also authored but work-in-progress by `Andrew Chen Wang`_), then the caching will work better since sharding +the cold/accessed-the-least records aren't invalidated as much. + +Cachalot is good when there are <50 updates per minute on a hot cached table. This is mostly due to cache invalidation. It's the same with any cache, +which is why we suggest you use cache-machine for hot caches. Cache-machine caches individual objects, taking up more in the memory store but +invalidates those individual objects instead of the entire table like cachalot. + +Yes, the bane of our entire existence lies in cache invalidation and naming variables. Why does cachalot suck when stuck with a huge table that's accessed rapidly? Since you've mixed your cold (90% of) with your hot (10% of) records, you're caching and invalidating an entire table. It's like trying to boil 1 ton of noodles inside ONE pot instead of 100 pots boiling 1 ton of noodles. Which is more efficient? The splitting up of them. + +Note 1: My personal experience with caches stems from Reddit's: https://redditblog.com/2017/01/17/caching-at-reddit/ + +Note 2: Technical comparison: https://django-cachalot.readthedocs.io/en/latest/introduction.html#comparison-with-similar-tools Discussion ---------- -On `our public gitter chat room `_. +Help? Technical chat? `It's here on Slack `_. + +Legacy chat: https://gitter.im/django-cachalot/Lobby + +.. _Andrew Chen Wang: https://github.com/Andrew-Chen-Wang \ No newline at end of file diff --git a/cachalot/apps.py b/cachalot/apps.py index 197bf28..b165063 100644 --- a/cachalot/apps.py +++ b/cachalot/apps.py @@ -13,7 +13,7 @@ from .settings import ( @register(Tags.compatibility) def check_django_version(app_configs, **kwargs): - if not (1, 11) <= django_version < (2, 2): + if not (1, 11) <= django_version < (3, 1): return [Error( 'Django %s is not compatible with this version of django-cachalot.' % django__version__, diff --git a/cachalot/utils.py b/cachalot/utils.py index f06738f..6aeb85b 100644 --- a/cachalot/utils.py +++ b/cachalot/utils.py @@ -12,7 +12,7 @@ from django.db import connections from django.db.models import QuerySet, Subquery, Exists from django.db.models.functions import Now from django.db.models.sql import Query, AggregateQuery -from django.db.models.sql.where import ExtraWhere, WhereNode +from django.db.models.sql.where import ExtraWhere, WhereNode, NothingNode from django.utils.six import text_type, binary_type, integer_types from .settings import ITERABLES, cachalot_settings @@ -108,6 +108,8 @@ def _find_subqueries_in_where(children): yield grand_child elif child_class is ExtraWhere: raise IsRawQuery + elif child_class is NothingNode: + pass else: rhs = child.rhs rhs_class = rhs.__class__ diff --git a/requirements.txt b/requirements.txt index 6e09420..ee092dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -Django>=1.11,<2.2 +Django>=1.11,<3.1 +tox==3.14.3 \ No newline at end of file