diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 9ff19fc..0000000 --- a/.coveragerc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -branch=True -source=dddp diff --git a/.gitignore b/.gitignore index 4d9c384..92c772f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,21 +2,30 @@ # $GIT_DIR/info/exclude or the core.excludesFile configuration variable as # described in https://git-scm.com/docs/gitignore -*.egg-info +# python *.pot *.py[co] __pycache__ + +# distutils/setuptools +.eggs/ +*.egg-info MANIFEST dist/ +build/ + +# docs docs/_build/ docs/locale/ -node_modules/ -tests/coverage_html/ -tests/.coverage -build/ -tests/report/ +docs/node_modules/ + +# test suite +.cache/ .coverage -.eggs/ .tox/ +htmlcov/ +tests/report/ + +# meteor dddp/test/build/ dddp/test/meteor_todos/.meteor/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..50fe7e8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +# .travis.yml automatically generated by ".travis.yml.sh" + +# Container-based builds used if "sudo: false" --> fast boot (1-6s) +# https://docs.travis-ci.com/user/ci-environment/ +sudo: false + +language: python + +env: + - TOXENV="py27-django1.8" + - TOXENV="py27-django1.9" + - TOXENV="py33-django1.8" + - TOXENV="py34-django1.8" + - TOXENV="py34-django1.9" + - TOXENV="py35-django1.8" + - TOXENV="py35-django1.9" + - TOXENV="pypy3-django1.8" + - TOXENV="pypy3-django1.9" + - TOXENV="pypy-django1.8" + - TOXENV="pypy-django1.9" + +install: + - pip install tox coveralls + +script: + - tox + +after_success: + coveralls diff --git a/.travis.yml.sh b/.travis.yml.sh new file mode 100755 index 0000000..89fde3d --- /dev/null +++ b/.travis.yml.sh @@ -0,0 +1,22 @@ +#!/bin/bash +cat< fast boot (1-6s) +# https://docs.travis-ci.com/user/ci-environment/ +sudo: false + +language: python + +env: +$( tox -l | grep '^py' | sort -n | sed -e 's/^.*$/ - TOXENV="\0"/' ) + +install: + - pip install tox coveralls + +script: + - tox + +after_success: + coveralls +EOF diff --git a/CHANGES.rst b/CHANGES.rst index 452281e..04ef82f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,34 +1,67 @@ Change Log ========== -0.18.1 +All notable changes to this project will be documented in this file. +This project adheres to `Semantic Versioning `_. + +0.19.1 ------ +* Dropped support for Django 1.7 (support expired on December 1 2015, + see https://www.djangoproject.com/download/#supported-versions). +* Require `setuptools>=18.5` at install time due to use of + `python_platform_implementation` environment marker. +* Moved repository to https://github.com/django-ddp/django-ddp (new + Github organisation). +* Started work on documentation refresh. +* Fail on start if any child threads can't start (eg: port in use). +* Rudimentary support for dynamic publications which may now refer to `this.user`. +* Correctly close DB connections during shutdown, useful for test suite. +* Add missing versions and dates to the change log, and note on Semantic + Versioning. +* Emit `django.core.signals.request_finished` to close DB connection and + yield to other greenlets between processing messages from WebSocket. +* Back to universal wheels, thanks to PEP-0496 environment markers. +* Expand test suite coverage to start a DDP server instance and perform + a HTTP GET request, no WebSocket/DDP tests yet. Fails in Pyton 3 + because of a bug in `gevent-websocket` + (https://bitbucket.org/noppo/gevent-websocket/issues/54/python-3-support). +* Fix for #23 (Python 3 compatibility). +* Set `application_name` on PostgreSQL async connection. +* Send `django.core.signals.request_finished` when closing WebSocket. +* Don't require `DJANGO_SETTINGS_MODULE` to import API. +* Tox test suite updated to runs against Python 2.7/3.3/3.4/3.5 and + Django 1.8/1.9. +* Build wheels from tox environment to ensure consistency. +* Remove `debug` argment from `PostgresGreenlet` class (unused). + +0.18.1 (2015-11-06) +------------------- * Don't assume Django projects include a `wsgi.py`. -0.18.0 ------- -* Python implementaiton specific builds using tox so that Python2 and +0.18.0 (2015-11-05) +------------------- +* Python implementation specific builds using tox so that Python2 and Python3 can have different dependencies (eg: gevent>=1.1 for Pyton3). * Added support for `METEOR_SETTINGS` environment variable. -0.17.3 ------- +0.17.3 (2015-10-30) +------------------- * Depend on gevent>=1.1b6 if running anything other than CPython 2.7, otherwise allow gevent 1.0 (current stable). * Preliminary (but broken) support for PyPy/Jython/IronPython though platform specific install_requires on psycopg2cffi instead of psycopg2 for all platforms except CPython 2/3. -0.17.2 ------- +0.17.2 (2015-10-29) +------------------- * Python 3 fixes using `six` compatibility library (#16, #17). -0.17.1 ------- +0.17.1 (2015-10-14) +------------------- * Fix minor issue where some subscription queries still used slow queries. -0.17.0 ------- +0.17.0 (2015-10-14) +------------------- * Make the SQL for subscriptions much faster for PostgreSQL. * Repeatable builds using ye olde make. * Use tox test runner - no tests yet (#11). @@ -36,15 +69,15 @@ Change Log * Started documentation using Sphinx (#10). * Python 3 style exception handling. -0.16.0 ------- +0.16.0 (2015-10-13) +------------------- * New setting: `DDP_API_ENDPOINT_DECORATORS`. This setting takes a list of dotted import paths to decorators which are applied to API endpoints. For example, enable New Relic instrumentation by adding the line below to your Django `settings.py`: .. code:: python DDP_API_ENDPOINT_DECORATORS = ['newrelic.agent.background_task'] - + * Fixed #7 -- Warn if using DB engines other than psycopg2 - thanks @Matvey-Kuk. * Improvements to error/exception handling. * Warn if many TX chunks are queued in case WebSocket has stalled. @@ -53,15 +86,15 @@ Change Log * Work towards #16 -- Use `psycopg2cffi` compatibility if `psycopg2` not installed. -0.15.0 ------- +0.15.0 (2015-09-25) +------------------- * Renamed `Logs` collection and publication to `dddp.logs` to be consistent with naming conventions used elsewhere. * Pass all attributes from `logging.LogRecord` via `dddp.logs` collection. * Use select_related() and resultant cached relational fields to speed up Colleciton.serialize() by significantly reducing round-trips to the database. * Fix bug in `get_meteor_ids()` which caused many extra database hits. -0.14.0 ------- +0.14.0 (2015-09-22) +------------------- * Correctly handle serving app content from the root path of a domain. * Account security tokens are now calculated for each minute allowing for finer grained token expiry. * Fix bug in error handling where invalid arguments were being passed to `logging.error()`. @@ -74,8 +107,8 @@ Change Log * Honour `--verbosity` in `dddp` command, now showing API endpoints in more verbose modes. * Updated `dddp.test` to Meteor 1.2 and also showing example of URL config to serve Meteor files from Python. -0.13.0 ------- +0.13.0 (2015-09-18) +------------------- * Abstract DDPLauncher out from dddp.main.serve to permit use from other contexts. * Allow Ctrl-C (Break) handling at any time. * Only run async DB connection when PostgresGreenlet is running. @@ -87,13 +120,13 @@ Change Log * Use sane default options for `python setup.py bdist_wheel`. * Fixed README link to meteor - thanks @LegoStormtroopr. -0.12.2 ------- +0.12.2 (2015-08-27) +------------------- * Set blank=True on AleaIdField, allowing adding items without inventing IDs yourself. -0.12.1 ------- +0.12.1 (2015-08-13) +------------------- * Add `AleaIdMixin` which provides `aid = AleaIdField(unique=True)` to models. * Use `AleaIdField(unique=True)` wherever possible when translating @@ -101,102 +134,102 @@ Change Log round trips to the database and hence drastically improving performance when such fields are available. -0.12.0 ------- +0.12.0 (2015-08-11) +------------------- * Get path to `star.json` from view config (defined in your urls.py) instead of from settings. * Dropped `dddp.server.views`, use `dddp.views` instead. -0.11.0 ------- +0.11.0 (2015-08-10) +------------------- * Support more than 8KB of change data by splitting large payloads into multiple chunks. -0.10.2 ------- +0.10.2 (2015-08-10) +------------------- * Add `Logs` publication that can be configured to emit logs via DDP through the use of the `dddp.logging.DDPHandler` log handler. * Add option to dddp daemon to provide a BackdoorServer (telnet) for interactive debugging (REPL) at runtime. -0.10.1 ------- +0.10.1 (2015-07-28) +------------------- * Bugfix dddp.accounts forgot_password feature. -0.10.0 ------- +0.10.0 (2015-07-21) +------------------- * Stop processing request middleware upon connection - see https://github.com/commoncode/django-ddp/commit/e7b38b89db5c4e252ac37566f626b5e9e1651a29 for rationale. Access to `this.request.user` is gone. * Add `this.user` handling to dddp.accounts. -0.9.14 ------- +0.9.14 (2015-07-18) +------------------- * Fix ordering of user added vs login ready in dddp.accounts authentication methods. -0.9.13 ------- +0.9.13 (2015-07-17) +------------------- * Add dddp.models.get_object_ids helper function. * Add ObjectMappingMixini abstract model mixin providing GenericRelation back to ObjectMapping model. -0.9.12 ------- +0.9.12 (2015-07-16) +------------------- * Bugfix /app.model/schema helper method on collections to work with more model field types. -0.9.11 ------- +0.9.11 (2015-07-14) +------------------- * Fix bug in post login/logout subscription handling. -0.9.10 ------- +0.9.10 (2015-07-08) +------------------- * Fix bug in Accounts.forgotPassword implementation. -0.9.9 ------ +0.9.9 (2015-07-08) +------------------ * Match return values for Accounts.changePassword and Accounts.changePassword methods in dddp.accounts submodule. -0.9.8 ------ +0.9.8 (2015-07-08) +------------------ * Fix method signature for Accouts.changePassword. -0.9.7 ------ +0.9.7 (2015-07-08) +------------------ * Updated Accounts hashing to prevent cross-purposing auth tokens. -0.9.6 ------ +0.9.6 (2015-07-07) +------------------ * Correct method signature to match Meteor Accounts.resetPassword in dddp.accounts submodule. -0.9.5 ------ +0.9.5 (2015-07-03) +------------------ * Include array of `permissions` on User publication. -0.9.4 ------ +0.9.4 (2015-06-29) +------------------ * Use mimetypes module to correctly guess mime types for Meteor files being served. -0.9.3 ------ +0.9.3 (2015-06-29) +------------------ * Include ROOT_URL_PATH_PREFIX in ROOT_URL when serving Meteor build files. -0.9.2 ------ +0.9.2 (2015-06-22) +------------------ * Use HTTPS for DDP URL if settings.SECURE_SSL_REDIRECT is set. -0.9.1 ------ +0.9.1 (2015-06-16) +------------------ * Added support for django.contrib.postres.fields.ArrayField serialization. -0.9.0 ------ +0.9.0 (2015-06-14) +------------------ * Added Django 1.8 compatibility. The current implementation has a hackish (but functional) implementation to use PostgreSQL's `array_agg` function. Pull requests are welcome. @@ -204,31 +237,31 @@ Change Log `dbarray` package for this even though not strictly required with Django 1.8. Once again, pull requests are welcome. -0.8.1 ------ +0.8.1 (2015-06-10) +------------------ * Add missing dependency on `pybars3` used to render boilerplate HTML template when serving Meteor application files. -0.8.0 ------ +0.8.0 (2015-06-09) +------------------ * Add `dddp.server` Django app to serve Meteor application files. * Show input params after traceback if exception occurs in API methods. * Small pylint cleanups. -0.7.0 ------ +0.7.0 (2015-05-28) +------------------ * Refactor serialization to improve performance through reduced number of database queries, especially on sub/unsub. * Fix login/logout user subscription, now emitting user `added`/ `removed` upon `login`/`logout` respectively. -0.6.5 ------ +0.6.5 (2015-05-27) +------------------ * Use OrderedDict for geventwebsocket.Resource spec to support geventwebsockets 0.9.4 and above. -0.6.4 ------ +0.6.4 (2015-05-27) +------------------ * Send `removed` messages when client unsubscribes from publications. * Add support for SSL options and --settings=SETTINGS args in dddp tool. * Add `optional` and `label` attributes to ManyToManyField simple @@ -237,17 +270,17 @@ Change Log than when queuing messages. * Move test projects into path that can be imported post install. -0.6.3 ------ +0.6.3 (2015-05-21) +------------------ * Refactor pub/sub functionality to fix support for `removed` messages. -0.6.2 ------ +0.6.2 (2015-05-20) +------------------ * Bugfix issue where DDP connection thread stops sending messages after changing item that has subscribers for other connections but not self. -0.6.1 ------ +0.6.1 (2015-05-18) +------------------ * Fix `createUser` method to login new user after creation. * Dump stack trace to console on error for easier debugging DDP apps. * Fix handing of F expressions in object change handler. @@ -255,14 +288,14 @@ Change Log * Per connection tracking of sent objects so changed/added sent appropriately. -0.6.0 ------ +0.6.0 (2015-05-12) +------------------ * Add dddp.accounts module which provides password based auth mapping to django.contrib.auth module. * Fix ordering of change messages and result message in method calls. -0.5.0 ------ +0.5.0 (2015-05-07) +------------------ * Drop relations to sessions.Session as WebSocket requests don't have HTTP cookie support -- **you must `migrate` your database after upgrading**. @@ -275,8 +308,8 @@ Change Log * Cleanup transaction handling to apply once at the entry point for DDP API calls. -0.4.0 ------ +0.4.0 (2015-04-28) +------------------ * Make live updates honour user_rel restrictions, also allow superusers to see everything. * Support serializing objects that are saved with F expressions by @@ -288,8 +321,8 @@ Change Log have user_rel items defined). This change includes a schema change, remember to run migrations after updating. -0.3.0 ------ +0.3.0 (2015-04-23) +------------------ * New DB field: Connection.server_addr -- **you must `migrate` your database after upgrading**. * Cleanup connections on shutdown (and purge associated subscriptions). @@ -300,15 +333,64 @@ Change Log * Fix `unsubscribe` from publications. * Fix `/schema` method call. -0.2.5 ------ +0.2.5 (2015-04-25) +------------------ * Fix foreign key references in change messages to correctly reference related object rather than source object. -0.2.4 ------ +0.2.4 (2015-04-15) +------------------ * Fix unicode rendering bug in DDP admin for ObjectMapping model. -0.2.3 ------ +0.2.3 (2015-04-15) +------------------ * Add `dddp` console script to start DDP service in more robust manner than using the dddp Django mangement command. + +0.2.2 (2015-04-14) +------------------ +* Don't include null/None reply from method calls in message. +* Force creation of Alea/Meteor ID even if nobody seems to care -- they + do care if they're using the ID with latency compensated views. +* Support collections to models having non-integer primary key fields. +* Fix latency compensated Alea/Meteor ID generation to match Meteor + semantics of using a namespace to generate seeded Alea PRNGs. + +0.2.1 (2015-04-10) +------------------ +* Change validation so that we now pass the DDP test suite + . +* Add lots of useful info to the README. + +0.2.0 (2015-04-08) +------------------ +* Add `dddp.models.get_meteor_id` and `dddp.models.get_object_id` + methods. +* Add `Connection`, `Subscription` and `SubscriptionColleciton` models, + instances of which are managed during life cycle of connections and + subscriptions. +* Fixed incorrect use of `django.core.serializers` where different + threads used same the serializer instance. +* Add `Collection.user_rel` class attribute allowing user-specific + filtering of objects at the collection level. +* Add `dddp.test` test project with example meteor-todos/django-ddp + project. +* Change `dddp` management command default port from 3000 to 8000. +* Validate `django.conf.settings.DATABASES` configuration on start. +* React to `django.db.models.signals.m2m_changed` model changes for + ManyToManyField. +* Add dependency on `django-dbarray`. + +0.1.1 (2015-03-11) +------------------ +* Add missing dependencies on `gevent`, `gevent-websocket`, + `meteor-ejson` and `psycogreen`. +* Meteor compatible latency compensation using Alea PRNG. +* Add `dddp.THREAD_LOCAL` with factories. +* Register django signals handlers via `AppConfig.ready()` handler. +* Add `dddp` management command. +* Add `dddp.models.AleaIdField` and `dddp.models.ObjectMapping` model. +* Major internal refactoring. + +0.1.0 (2015-02-13) +------------------ +* Working proof-of-concept. diff --git a/MANIFEST.in b/MANIFEST.in index 8a0b3c1..377c556 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,11 +2,15 @@ include LICENSE include *.rst include *.sh include *.txt +include requirements*.txt include .gitignore -include .coveragerc include Makefile +exclude tox.ini +graft dddp/test/meteor_todos +prune dddp/test/build +prune dddp/test/meteor_todos/.meteor/local graft docs prune docs/_build -include tox.ini -graft dddp/test/meteor_todos -prune dddp/test/meteor_todos/.meteor +prune docs/node_modules +exclude .travis.yml.sh +exclude .travis.yml diff --git a/Makefile b/Makefile index 8b2e277..91e6e1b 100644 --- a/Makefile +++ b/Makefile @@ -2,49 +2,53 @@ NAME := $(shell python setup.py --name) VERSION := $(shell python setup.py --version) SDIST := dist/${NAME}-${VERSION}.tar.gz -WHEEL_PY2 := dist/$(subst -,_,${NAME})-${VERSION}-py2-none-any.whl -WHEEL_PY3 := dist/$(subst -,_,${NAME})-${VERSION}-py3-none-any.whl -WHEEL_PYPY := dist/$(subst -,_,${NAME})-${VERSION}-pypy-none-any.whl +WHEEL := dist/$(subst -,_,${NAME})-${VERSION}-py2.py3-none-any.whl .PHONY: all test clean clean-docs clean-dist upload-docs upload-pypi dist .INTERMEDIATE: dist.intermediate docs -all: docs dist +all: .travis.yml docs dist test: tox -vvv -clean: clean-docs clean-dist +clean: clean-docs clean-dist clean-pyc clean-docs: $(MAKE) -C docs/ clean clean-dist: - rm -f "${SDIST}" "${WHEEL_PY2}" "${WHEEL_PY3}" + rm -rf "${SDIST}" "${WHEEL}" dddp/test/build/ dddp/test/meteor_todos/.meteor/local/ + +clean-pyc: + find . -type f -name \*.pyc -print0 | xargs -0 rm -f docs: $(shell find docs/ -type f -name \*.rst) docs/conf.py docs/Makefile $(shell find docs/_static/ -type f) $(shell find docs/_templates/ -type f) README.rst CHANGES.rst $(MAKE) -C docs/ clean html touch "$@" -dist: ${SDIST} ${WHEEL_PY2} ${WHEEL_PY3} +dist: ${SDIST} ${WHEEL} + @echo 'Build successful, `${MAKE} upload` when ready to release.' ${SDIST}: dist.intermediate + @echo "Testing ${SDIST}..." + tox --notest --installpkg ${SDIST} -${WHEEL_PY2}: dist.intermediate - -${WHEEL_PY3}: dist.intermediate - -${WHEEL_PYPY}: - tox -e pypy-test-dist +${WHEEL}: dist.intermediate + @echo "Testing ${WHEEL}..." + tox --notest --installpkg ${WHEEL} dist.intermediate: $(shell find dddp -type f) - tox -e py27-test-dist,py34-test-dist + tox -e dist upload: upload-pypi upload-docs -upload-pypi: ${SDIST} ${WHEEL_PY2} ${WHEEL_PY3} - twine upload "${WHEEL_PY2}" "${WHEEL_PY3}" "${SDIST}" +upload-pypi: ${SDIST} ${WHEEL} + twine upload "${WHEEL}" "${SDIST}" upload-docs: docs/_build/ python setup.py upload_sphinx --upload-dir="$ "$@" diff --git a/README.rst b/README.rst index da945eb..58e64ab 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,4 @@ +========== Django DDP ========== @@ -259,6 +260,7 @@ Contributors This project is forever grateful for the love, support and respect given by the awesome team at `Common Code`_. +.. _Django DDP: https://github.com/django-ddp/django-ddp .. _Django: https://www.djangoproject.com/ .. _Django signals: https://docs.djangoproject.com/en/stable/topics/signals/ .. _Common Code: https://commoncode.com.au/ diff --git a/dddp/__init__.py b/dddp/__init__.py index 5978f8c..e027d7f 100644 --- a/dddp/__init__.py +++ b/dddp/__init__.py @@ -1,11 +1,11 @@ -"""Django/PostgreSQL implementation of the Meteor DDP service.""" +"""Django/PostgreSQL implementation of the Meteor server.""" from __future__ import unicode_literals -import os.path import sys from gevent.local import local from dddp import alea -__version__ = '0.18.1' +__version__ = '0.19.0' +__url__ = 'https://github.com/django-ddp/django-ddp' default_app_config = 'dddp.apps.DjangoDDPConfig' @@ -121,6 +121,9 @@ THREAD_LOCAL_FACTORIES = { 'alea_random': alea.Alea, 'random_streams': RandomStreams, 'serializer': serializer_factory, + 'user_id': lambda: None, + 'user_ddp_id': lambda: None, + 'user': lambda: None, } THREAD_LOCAL = ThreadLocal() METEOR_ID_CHARS = u'23456789ABCDEFGHJKLMNPQRSTWXYZabcdefghijkmnopqrstuvwxyz' diff --git a/dddp/api.py b/dddp/api.py index fc0a745..0e77f47 100644 --- a/dddp/api.py +++ b/dddp/api.py @@ -10,9 +10,8 @@ import uuid # requirements import dbarray from django.conf import settings -from django.contrib.auth import get_user_model import django.contrib.postgres.fields -from django.db import connections, router +from django.db import connections, router, transaction from django.db.models import aggregates, Q try: # pylint: disable=E0611 @@ -108,6 +107,7 @@ def api_endpoint(path_or_func=None, decorate=True): Examples: + >>> from dddp.api import APIMixin, api_endpoint >>> class Counter(APIMixin): ... value = 0 ... @@ -270,6 +270,13 @@ class Collection(APIMixin): queryset = property(get_queryset) + @property + def user_model(self): + """Cached property getter around `get_user_model`.""" + from django.contrib.auth import get_user_model + val = self.__dict__['user_model'] = get_user_model() + return val + def objects_for_user(self, user, qs=None, xmin__lte=None): """Find objects in queryset related to specified user.""" qs = self.get_queryset(qs) @@ -329,7 +336,7 @@ class Collection(APIMixin): if self.always_allow_superusers: user_ids.update( - get_user_model().objects.filter( + self.user_model.objects.filter( is_superuser=True, is_active=True, ).values_list('pk', flat=True) ) @@ -531,15 +538,40 @@ class Publication(APIMixin): name = None queries = None - def get_queries(self, *params): - """DDP get_queries - must override if using params.""" - if params: - raise NotImplementedError( - 'Publication params not implemented on %r publication.' % ( - self.name, - ), - ) - return self.queries[:] + def user_queries(self, user, *params): + """Return queries for this publication as seen by `user`.""" + try: + get_queries = self.get_queries + except AttributeError: + # statically defined queries + if self.queries is None: + raise NotImplementedError( + 'Must set either queries or implement get_queries method.', + ) + if params: + raise NotImplementedError( + 'Publication params not implemented on %r publication.' % ( + self.name, + ), + ) + return self.queries[:] + + if user is False: + # no need to play with `this.user_id` or `this.user_ddp_id`. + return get_queries(*params) + + # stash the old user details + old_user_id = this.user_id + old_user_ddp_id = this.user_ddp_id + # apply the desired user details + this.user_id = None if user is None else user.pk + this.user_ddp_id = None if user is None else get_meteor_id(user) + try: + return get_queries(*params) + finally: + # restore the old user details + this.user_id = old_user_id + this.user_ddp_id = old_user_ddp_id @api_endpoint def collections(self, *params): @@ -548,7 +580,7 @@ class Publication(APIMixin): set( hasattr(qs, 'model') and model_name(qs.model) or qs[1] for qs - in self.get_queries(*params) + in self.get_queries(False, *params) ) ) @@ -604,7 +636,7 @@ class DDP(APIMixin): (col, qs) for (qs, col) in ( self.qs_and_collection(qs) for qs - in pub.get_queries(*params) + in pub.user_queries(obj.user, *params) ) ) # mergebox via MVCC! For details on how this is possible, read this: @@ -628,7 +660,7 @@ class DDP(APIMixin): pk=obj.pk, ).order_by('pk').distinct(): other_pub = self.get_pub_by_name(other.publication) - for qs in other_pub.get_queries(*other.params): + for qs in other_pub.user_queries(other.user, *other.params): qs, col = self.qs_and_collection(qs) if col not in to_send: continue @@ -645,8 +677,21 @@ class DDP(APIMixin): @api_endpoint def sub(self, id_, name, *params): """Create subscription, send matched objects that haven't been sent.""" - return self.do_sub(id_, name, False, *params) + try: + return self.do_sub(id_, name, False, *params) + except Exception as err: + this.send({ + 'msg': 'nosub', + 'id': id_, + 'error': { + 'error': 500, + 'errorType': 'Meteor.Error', + 'message': '%s' % err, + 'reason': 'Subscription failed', + }, + }) + @transaction.atomic def do_sub(self, id_, name, silent, *params): """Subscribe the current thread to the specified publication.""" try: @@ -655,6 +700,7 @@ class DDP(APIMixin): if not silent: this.send({ 'msg': 'nosub', + 'id': id_, 'error': { 'error': 404, 'errorType': 'Meteor.Error', @@ -915,10 +961,14 @@ class DDP(APIMixin): collections__model_name=model_name(model), ).prefetch_related('collections'): pub = self.get_pub_by_name(sub.publication) + try: + queries = list(pub.user_queries(sub.user, *sub.params)) + except Exception: + queries = [] for qs, col in ( self.qs_and_collection(qs) for qs - in pub.get_queries(*sub.params) + in queries ): # check if obj is an instance of the model for the queryset if qs.model is not model: diff --git a/dddp/main.py b/dddp/main.py index 4de20f2..dd8d12c 100644 --- a/dddp/main.py +++ b/dddp/main.py @@ -103,9 +103,7 @@ class DDPLauncher(object): # shutdown existing connections close_old_connections() - DDPLauncher.pgworker = PostgresGreenlet( - connection, debug=debug, - ) + DDPLauncher.pgworker = PostgresGreenlet(connection) # use settings.WSGI_APPLICATION or fallback to default Django WSGI app from django.conf import settings @@ -189,6 +187,9 @@ class DDPLauncher(object): for server in self.servers + [DDPLauncher.pgworker]: self.logger.debug('Stopping %s', server) server.stop() + # wait for all threads to stop. + gevent.joinall(self.threads + [DDPLauncher.pgworker]) + self.threads = [] def start(self): """Run PostgresGreenlet and web/debug servers.""" @@ -204,7 +205,12 @@ class DDPLauncher(object): self.print('=> Started PostgresGreenlet.') for server in self.servers: thread = gevent.spawn(server.serve_forever) + gevent.sleep() # yield to thread in case it can't start self.threads.append(thread) + if thread.dead: + # thread died, stop everything and re-raise the exception. + self.stop() + thread.get() if isinstance(server, geventwebsocket.WebSocketServer): self.print( '=> App running at: %s://%s:%d/' % ( @@ -229,6 +235,7 @@ class DDPLauncher(object): self._stop_event.wait() # wait for all threads to stop. gevent.joinall(self.threads + [DDPLauncher.pgworker]) + self.threads = [] def addr(val, default_port=8000, defualt_host='localhost'): diff --git a/dddp/models.py b/dddp/models.py index 24357bc..28c52e1 100644 --- a/dddp/models.py +++ b/dddp/models.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import collections import os -from django.db import models, transaction +from django.db import models from django.db.models.fields import NOT_PROVIDED from django.conf import settings from django.contrib.contenttypes.fields import ( diff --git a/dddp/postgres.py b/dddp/postgres.py index 603309f..ef8718a 100644 --- a/dddp/postgres.py +++ b/dddp/postgres.py @@ -6,15 +6,17 @@ import ejson import gevent import gevent.queue import gevent.select +import os import psycopg2 # green import psycopg2.extensions +import socket class PostgresGreenlet(gevent.Greenlet): """Greenlet for multiplexing database operations.""" - def __init__(self, conn, debug=False): + def __init__(self, conn): """Prepare async connection.""" super(PostgresGreenlet, self).__init__() import logging @@ -33,15 +35,41 @@ class PostgresGreenlet(gevent.Greenlet): def _run(self): # pylint: disable=method-hidden """Spawn sub tasks, wait for stop signal.""" conn_params = self.connection.get_connection_params() + # See http://initd.org/psycopg/docs/module.html#psycopg2.connect and + # http://www.postgresql.org/docs/current/static/libpq-connect.html + # section 31.1.2 (Parameter Key Words) for details on available params. conn_params.update( async=True, + application_name='{} pid={} django-ddp'.format( + socket.gethostname(), # hostname + os.getpid(), # PID + )[:64], # 64 characters for default PostgreSQL build config ) - conn = psycopg2.connect(**conn_params) + conn = None + while conn is None: + try: + conn = psycopg2.connect(**conn_params) + except psycopg2.OperationalError as err: + # Some variants of the psycopg2 driver for Django add extra + # params that aren't meant to be passed directly to + # `psycopg2.connect()` -- issue a warning and try again. + msg = ('%s' % err).strip() + msg_prefix = 'invalid connection option "' + if not msg.startswith(msg_prefix): + # *waves hand* this is not the errror you are looking for. + raise + key = msg[len(msg_prefix):-1] + self.logger.warning( + 'Ignoring unknown settings.DATABASES[%r] option: %s=%r', + self.connection.alias, + key, conn_params.pop(key), + ) self.poll(conn) # wait for conneciton to start - cur = conn.cursor() import logging logging.getLogger('dddp').info('=> Started PostgresGreenlet.') + + cur = conn.cursor() cur.execute('LISTEN "ddp";') while not self._stop_event.is_set(): try: @@ -55,7 +83,9 @@ class PostgresGreenlet(gevent.Greenlet): finally: self.select_greenlet = None self.poll(conn) + self.poll(conn) cur.close() + self.poll(conn) conn.close() def stop(self): diff --git a/dddp/test/__init__.py b/dddp/test/__init__.py index e69de29..8740a3b 100644 --- a/dddp/test/__init__.py +++ b/dddp/test/__init__.py @@ -0,0 +1,17 @@ +# This file mainly exists to allow `python setup.py test` to work. +import os +import sys + +import dddp +import django +from django.test.utils import get_runner +from django.conf import settings + + +def run_tests(): + os.environ['DJANGO_SETTINGS_MODULE'] = 'dddp.test.test_project.settings' + dddp.greenify() + django.setup() + test_runner = get_runner(settings)() + failures = test_runner.run_tests(['dddp', 'dddp.test.django_todos']) + sys.exit(bool(failures)) diff --git a/dddp/test/django_todos/tests.py b/dddp/test/django_todos/tests.py index 7ce503c..70500dc 100644 --- a/dddp/test/django_todos/tests.py +++ b/dddp/test/django_todos/tests.py @@ -1,3 +1,34 @@ -from django.test import TestCase +"""Django Todos test suite.""" -# Create your tests here. +import doctest +import os +import unittest + +os.environ['DJANGO_SETTINGS_MODULE'] = 'dddp.test.test_project.settings' + +DOCTEST_MODULES = [ +] + + +class NoOpTest(unittest.TestCase): + def test_noop(self): + assert True + + +def load_tests(loader, tests, pattern): + """Specify which test cases to run.""" + del pattern + suite = unittest.TestSuite() + # add all TestCase classes from this (current) module + for attr in globals().values(): + try: + if not issubclass(attr, unittest.TestCase): + continue # not subclass of TestCase + except TypeError: + continue # not a class + tests = loader.loadTestsFromTestCase(attr) + suite.addTests(tests) + # add doctests defined in DOCTEST_MODULES + for doctest_module in DOCTEST_MODULES: + suite.addTest(doctest.DocTestSuite(doctest_module)) + return suite diff --git a/dddp/test/test_project/settings.py b/dddp/test/test_project/settings.py index 8e697a3..781c342 100644 --- a/dddp/test/test_project/settings.py +++ b/dddp/test/test_project/settings.py @@ -62,7 +62,11 @@ WSGI_APPLICATION = 'dddp.test.test_project.wsgi.application' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'django_ddp_test_project', + 'NAME': os.environ.get('PGDATABASE', 'django_ddp_test_project'), + 'USER': os.environ.get('PGUSER', os.environ['LOGNAME']), + 'PORT': int(os.environ.get('PGPORT', '0')) or None, + 'PASSWORD': os.environ.get('PGPASSWORD', '') or None, + 'HOST': os.environ.get('PGHOST', '') or None, } } diff --git a/dddp/tests.py b/dddp/tests.py new file mode 100644 index 0000000..3f98d06 --- /dev/null +++ b/dddp/tests.py @@ -0,0 +1,99 @@ +"""Django DDP test suite.""" +from __future__ import unicode_literals + +import doctest +import errno +import os +import socket +import unittest +from django.test import TestCase +import dddp.alea +from dddp.main import DDPLauncher +# pylint: disable=E0611, F0401 +from six.moves.urllib_parse import urljoin + +os.environ['DJANGO_SETTINGS_MODULE'] = 'dddp.test.test_project.settings' + +DOCTEST_MODULES = [ + dddp.alea, +] + + +class DDPServerTestCase(TestCase): + + """Test case that starts a DDP server.""" + + server_addr = '127.0.0.1' + server_port_range = range(8000, 8080) + ssl_certfile_path = None + ssl_keyfile_path = None + + def setUp(self): + """Fire up the DDP server.""" + self.server_port = 8000 + kwargs = {} + if self.ssl_certfile_path: + kwargs['certfile'] = self.ssl_certfile_path + if self.ssl_keyfile_path: + kwargs['keyfile'] = self.ssl_keyfile_path + self.scheme = 'https' if kwargs else 'http' + for server_port in self.server_port_range: + self.server = DDPLauncher(debug=True) + self.server.add_web_servers( + [ + (self.server_addr, server_port), + ], + **kwargs + ) + try: + self.server.start() + self.server_port = server_port + return # server started + except socket.error as err: + if err.errno != errno.EADDRINUSE: + raise # error wasn't "address in use", re-raise. + continue # port in use, try next port. + raise RuntimeError('Failed to start DDP server.') + + def tearDown(self): + """Shut down the DDP server.""" + self.server.stop() + + def url(self, path): + """Return full URL for given path.""" + return urljoin( + '%s://%s:%d' % (self.scheme, self.server_addr, self.server_port), + path, + ) + + +class LaunchTestCase(DDPServerTestCase): + + """Test that server launches and handles GET request.""" + + def test_get(self): + """Perform HTTP GET.""" + import requests + resp = requests.get(self.url('/')) + self.assertEqual(resp.status_code, 200) + + +def load_tests(loader, tests, pattern): + """Specify which test cases to run.""" + del pattern + suite = unittest.TestSuite() + # add all TestCase classes from this (current) module + for attr in globals().values(): + if attr is DDPServerTestCase: + continue # not meant to be executed, is has no tests. + try: + if not issubclass(attr, unittest.TestCase): + continue # not subclass of TestCase + except TypeError: + continue # not a class + tests = loader.loadTestsFromTestCase(attr) + suite.addTests(tests) + # add doctests defined in DOCTEST_MODULES + for doctest_module in DOCTEST_MODULES: + suite.addTest(doctest.DocTestSuite(doctest_module)) + return suite diff --git a/dddp/views.py b/dddp/views.py index dbb72a0..648a7ca 100644 --- a/dddp/views.py +++ b/dddp/views.py @@ -3,7 +3,6 @@ from __future__ import absolute_import, unicode_literals from copy import deepcopy import io -import logging import mimetypes import os.path @@ -16,18 +15,22 @@ import pybars # from https://www.xormedia.com/recursively-merge-dictionaries-in-python/ -def dict_merge(a, b): - '''recursively merges dict's. not just simple a['key'] = b['key'], if - both a and bhave a key who's value is a dict then dict_merge is called - on both values and the result stored in the returned dictionary.''' - if not isinstance(b, dict): - return b - result = deepcopy(a) - for k, v in b.iteritems(): - if k in result and isinstance(result[k], dict): - result[k] = dict_merge(result[k], v) +def dict_merge(lft, rgt): + """ + Recursive dict merge. + + Recursively merges dict's. not just simple lft['key'] = rgt['key'], if + both lft and rgt have a key who's value is a dict then dict_merge is + called on both values and the result stored in the returned dictionary. + """ + if not isinstance(rgt, dict): + return rgt + result = deepcopy(lft) + for key, val in rgt.iteritems(): + if key in result and isinstance(result[key], dict): + result[key] = dict_merge(result[key], val) else: - result[k] = deepcopy(v) + result[key] = deepcopy(val) return result @@ -54,16 +57,11 @@ class MeteorView(View): """Django DDP Meteor server view.""" - logger = logging.getLogger(__name__) http_method_names = ['get', 'head'] json_path = None runtime_config = None - manifest = None - program_json = None - program_json_path = None - star_json = None # top level layout url_map = None @@ -82,7 +80,7 @@ class MeteorView(View): """ Initialisation for Django DDP server view. - `Meteor.settings` is sourced from the following (later take precedence): + The following items populate `Meteor.settings` (later take precedence): 1. django.conf.settings.METEOR_SETTINGS 2. os.environ['METEOR_SETTINGS'] 3. MeteorView.meteor_settings (class attribute) or empty dict @@ -96,16 +94,14 @@ class MeteorView(View): 4. MeteorView.as_view(meteor_public_envs=...) """ self.runtime_config = {} - self.meteor_settings = reduce( - dict_merge, - [ + self.meteor_settings = {} + for other in [ getattr(settings, 'METEOR_SETTINGS', {}), loads(os.environ.get('METEOR_SETTINGS', '{}')), self.meteor_settings or {}, kwargs.pop('meteor_settings', {}), - ], - {}, - ) + ]: + self.meteor_settings = dict_merge(self.meteor_settings, other) self.meteor_public_envs = set() self.meteor_public_envs.update( getattr(settings, 'METEOR_PUBLIC_ENVS', []), @@ -251,18 +247,14 @@ class MeteorView(View): def get(self, request, path): """Return HTML (or other related content) for Meteor.""" - self.logger.debug( - '[%s:%s] %s %s %s', - request.META['REMOTE_ADDR'], request.META['REMOTE_PORT'], - request.method, request.path, - request.META['SERVER_PROTOCOL'], - ) if path == 'meteor_runtime_config.js': config = { 'DDP_DEFAULT_CONNECTION_URL': request.build_absolute_uri('/'), 'PUBLIC_SETTINGS': self.meteor_settings.get('public', {}), 'ROOT_URL': request.build_absolute_uri( - '%s/' % self.runtime_config.get('ROOT_URL_PATH_PREFIX', ''), + '%s/' % ( + self.runtime_config.get('ROOT_URL_PATH_PREFIX', ''), + ), ), 'ROOT_URL_PATH_PREFIX': '', } diff --git a/dddp/websocket.py b/dddp/websocket.py index 6d098c5..b6ecfa0 100644 --- a/dddp/websocket.py +++ b/dddp/websocket.py @@ -12,7 +12,9 @@ import traceback from six.moves import range as irange import ejson +import gevent import geventwebsocket +from django.core import signals from django.core.handlers.base import BaseHandler from django.core.handlers.wsgi import WSGIRequest from django.db import connection, transaction @@ -151,6 +153,7 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): del self.pgworker.connections[self.connection.pk] self.connection.delete() self.connection = None + signals.request_finished.send(sender=self.__class__) self.logger.info('- %s %s', self, args or 'CLOSE') def on_message(self, message): @@ -195,6 +198,11 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): except Exception as err: traceback.print_exc() self.error(err) + # emit request_finished signal to close DB connections + signals.request_finished.send(sender=self.__class__) + if msgs: + # yield to other greenlets before processing next msg + gevent.sleep() except geventwebsocket.WebSocketError as err: self.ws.close() diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 101a9b6..0000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ --r requirements.txt -Sphinx==1.3.1 -Sphinx-PyPI-upload==0.2.1 -cloud_sptheme==1.7 -twine==1.6.4 diff --git a/docs/Makefile b/docs/Makefile index 1912130..9a8d4c6 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -4,7 +4,7 @@ # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build -PAPER = +PAPER = a4 BUILDDIR = _build # User-friendly check for sphinx-build @@ -175,3 +175,13 @@ pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +pdf: + $(SPHINXBUILD) -b pdf $(ALLSPHINXOPTS) _build/pdf + @echo + @echo "Build finished. The PDF files are in _build/pdf." + +dash: + $(SPHINXBUILD) -b dash $(ALLSPHINXOPTS) _build/dash + @echo + @echo "Build finished. The DASH files are in _build/dash." diff --git a/docs/_static/django-ddp-logo.png b/docs/_static/django-ddp-logo.png new file mode 100644 index 0000000..c91f6ab Binary files /dev/null and b/docs/_static/django-ddp-logo.png differ diff --git a/docs/admin/index.rst b/docs/admin/index.rst new file mode 100644 index 0000000..58e50fd --- /dev/null +++ b/docs/admin/index.rst @@ -0,0 +1,43 @@ +*********************** +Administration Handbook +*********************** + +This section is aimed at DevOps and project administrators to assist in +installing and maintaining a site using Django DDP. + +Requirements +============ + +You must be using PostgreSQL_ with psycopg2_ in your Django_ project for +django-ddp to work. There is no requirement on any asynchronous +framework such as Reddis or crossbar.io as they are simply not needed +given the asynchronous support provided by PostgreSQL_ with psycopg2_. + + +Installation +============ + +Install the latest release from pypi (recommended): + +.. code:: sh + + pip install django-ddp + +Don't forget to add `dddp` to your `requirements.txt` and/or the +`install_requires` section in `setup.py` for your project as necessary. + +Clone and use development version direct from GitHub to test pre-release +code (no GitHub account required): + +.. code:: sh + + pip install -e + git+https://github.com/commoncode/django-ddp@develop#egg=django-ddp + + +.. _Django: https://www.djangoproject.com/ +.. _Django signals: https://docs.djangoproject.com/en/stable/topics/signals/ +.. _Gevent: http://www.gevent.org/ +.. _PostgreSQL: http://postgresql.org/ +.. _psycopg2: http://initd.org/psycopg/ +.. _WebSockets: http://www.w3.org/TR/websockets/ diff --git a/docs/changelog.rst b/docs/changelog.rst deleted file mode 100644 index f6a3529..0000000 --- a/docs/changelog.rst +++ /dev/null @@ -1,3 +0,0 @@ -.. _changelog: - -.. include:: ../CHANGES.rst diff --git a/docs/conf.py b/docs/conf.py index a153f23..d60fa22 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,7 +15,29 @@ import sys import os -import cloud_sptheme as csp +import django +from django.conf import settings + +settings.configure( + INSTALLED_APPS=[ + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'dddp', + ], + DATABASES={ + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': os.environ.get('PGDATABASE', 'django_ddp_docs_project'), + 'USER': os.environ.get('PGUSER', os.environ['LOGNAME']), + 'PORT': int(os.environ.get('PGPORT', '0')) or None, + 'PASSWORD': os.environ.get('PGPASSWORD', '') or None, + 'HOST': os.environ.get('PGHOST', '') or None, + }, + }, +) +django.setup() + +import alabaster # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -32,13 +54,19 @@ import cloud_sptheme as csp # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.napoleon', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', - 'cloud_sptheme.ext.index_styling', - 'cloud_sptheme.ext.relbar_toc', + 'rst2pdf.pdfbuilder', + 'sphinxcontrib.dashbuilder', + 'alabaster', ] +# Show `.. todo::` items in output. +todo_include_todos = True + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -60,9 +88,9 @@ copyright = u'2015, Tyson Clugg' # built documents. # # The short X.Y version. -version = '0.18' +version = '0.19' # The full version, including alpha/beta/rc tags. -release = '0.18.1' +release = '0.19.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -107,17 +135,22 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'cloud' +html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - 'max_width': '80em', + 'logo': 'django-ddp-logo.png', + 'github_user': 'django-ddp', + 'github_repo': 'django-ddp', + 'github_button': 'true', + 'github_type': 'star', + 'github_banner': 'true', } # Add any paths that contain custom themes here, relative to this directory. -html_theme_path = [csp.get_theme_dir()] +html_theme_path = [alabaster.get_path()] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". @@ -154,7 +187,15 @@ html_static_path = ['_static'] #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +html_sidebars = { + '**': [ + 'about.html', + 'navigation.html', + 'relations.html', + 'searchbox.html', + 'donate.html', + ], +} # Additional templates that should be rendered to pages, maps page names to # template names. @@ -267,3 +308,21 @@ texinfo_documents = [ # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False + +# Grouping the document tree into PDF files. List of tuples +# (source start file, target name, title, author, options). +# +# If there is more than one author, separate them with \\. +# For example: r'Guido van Rossum\\Fred L. Drake, Jr., editor' +# +# The options element is a dictionary that lets you override +# this config per-document. +# For example, +# ('index', u'MyProject', u'My Project', u'Author Name', +# dict(pdf_compressed = True)) +# would mean that specific document would be compressed +# regardless of the global pdf_compressed setting. + +pdf_documents = [ + ('changes', u'django-ddp', u'django-ddp', u'Tyson Clugg'), +] diff --git a/docs/devel/api/dddp.api.rst b/docs/devel/api/dddp.api.rst new file mode 100644 index 0000000..0ec6ef4 --- /dev/null +++ b/docs/devel/api/dddp.api.rst @@ -0,0 +1,5 @@ +dddp.api +-------- + +.. automodule:: dddp.api + :members: diff --git a/docs/devel/api/index.rst b/docs/devel/api/index.rst new file mode 100644 index 0000000..39d6b25 --- /dev/null +++ b/docs/devel/api/index.rst @@ -0,0 +1,8 @@ +============= +API Reference +============= + +.. toctree:: + :glob: + + * diff --git a/docs/devel/index.rst b/docs/devel/index.rst new file mode 100644 index 0000000..74de317 --- /dev/null +++ b/docs/devel/index.rst @@ -0,0 +1,7 @@ +=================== +Developers Handbook +=================== + +.. toctree:: + + api/index diff --git a/docs/gulpfile.js b/docs/gulpfile.js new file mode 100644 index 0000000..cc4a95c --- /dev/null +++ b/docs/gulpfile.js @@ -0,0 +1,22 @@ +var exec = require('child_process').exec; +var gulp = require('gulp'); +var browserSync = require('browser-sync'); + +gulp.task('sphinx', function(cb) { + exec('make html', function(err, stdout, stderr) { + console.log(stdout); + console.log(stderr); + cb(err); + browserSync.reload(); + }); +}); + +gulp.task('default', ['sphinx'], function() { + browserSync({ + open: false, + server: { + baseDir: '_build/html/' + } + }); + gulp.watch(["../README.rst", "../LICENSE", "../CHANGES.rst", "**/*.rst", "_static/**", "conf.py"], ['sphinx']); +}); diff --git a/docs/index.rst b/docs/index.rst index 001c5eb..05a7927 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,51 +1,31 @@ -.. Django DDP documentation master file, created by - sphinx-quickstart on Tue Oct 13 23:24:39 2015. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +########## +Django DDP +########## -.. include:: ../README.rst +.. image:: _static/django-ddp-logo.png + :alt: django-ddp — reactive ★ realtime ★ relational ★ robust + :align: center -Installation: -------------- -`Django DDP`_ is available for installation direct from PyPi_: - -.. code:: bash - - pip install django-ddp - -Links ------ - -.. image:: https://readthedocs.org/projects/django-ddp/badge/?version=latest - :target: https://readthedocs.org/projects/django-ddp/?badge=latest - :alt: Documentation Status - :align: right - -The latest documentation is available online at -https://django-ddp.readthedocs.org/. - -Source code is available online at https://github.com/commoncode/django-ddp. +.. rubric:: + Django DDP lets you create websites and mobile apps that are + reactive, realtime, relational & robust -- using Django and + Meteor together with the smallest server (and environmental) + footprint. Contents -------- .. toctree:: - :glob: - :maxdepth: 1 + :maxdepth: 2 - changelog - django-ddp* - - -Indices and tables -================== + readme + tutorial/index + admin/index + devel/index + reference/index * :ref:`genindex` * :ref:`modindex` * :ref:`search` -License -======= -.. include:: ../LICENSE - .. _pypi: https://pypi.python.org/pypi/django-ddp diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 0000000..4dc0eb5 --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1,3 @@ +.. _readme: + +.. include:: ../README.rst diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst new file mode 100644 index 0000000..cca3726 --- /dev/null +++ b/docs/reference/changelog.rst @@ -0,0 +1,3 @@ +.. _changelog: + +.. include:: ../../CHANGES.rst diff --git a/docs/reference/index.rst b/docs/reference/index.rst new file mode 100644 index 0000000..2b4f81b --- /dev/null +++ b/docs/reference/index.rst @@ -0,0 +1,9 @@ +========= +Reference +========= + +.. toctree:: + :maxdepth: 1 + + changelog + license diff --git a/docs/reference/license.rst b/docs/reference/license.rst new file mode 100644 index 0000000..3547d34 --- /dev/null +++ b/docs/reference/license.rst @@ -0,0 +1,5 @@ +License +------- + +.. include:: ../../LICENSE + :literal: diff --git a/docs/tutorial/index.rst b/docs/tutorial/index.rst new file mode 100644 index 0000000..91d2282 --- /dev/null +++ b/docs/tutorial/index.rst @@ -0,0 +1,12 @@ +======== +Tutorial +======== + +.. todo:: This documentation is incomplete -- pull requests are welcome! + +* Install Python / virtualenv / Meteor +* Setup repo / add basics: + - setup.py + - django-admin.py startproject + - meteor create + - letsencrypt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..bade6b9 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +# things you need to build from source and distribute a release +Sphinx==1.3.3 +Sphinx-PyPI-upload==0.2.1 +twine==1.6.4 +sphinxcontrib-dashbuilder==0.1.0 +rst2pdf==0.93 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..a5c10fa --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,2 @@ +# things required to run test suite +requests==2.9.0 diff --git a/requirements.txt b/requirements.txt index e0617b0..7a314e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ -Django==1.8.5 -gevent==1.0.2 +Django>=1.7 +gevent==1.0.2 ; platform_python_implementation == "CPython" and python_version < "3.0" +gevent==1.1rc2 ; platform_python_implementation != "CPython" or python_version >= "3.0" gevent-websocket==0.9.5 -psycopg2==2.6.1 +psycopg2==2.6.1 ; platform_python_implementation == "CPython" +psycopg2cffi>=2.7.2 ; platform_python_implementation != "CPython" six==1.10.0 diff --git a/setup.cfg b/setup.cfg index aa76bae..3c6e79c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [bdist_wheel] -universal=0 +universal=1 diff --git a/setup.py b/setup.py index cf5b95f..260f9bc 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,43 @@ #!/usr/bin/env python -"""Django/PostgreSQL implementation of the Meteor DDP service.""" -import platform -import sys -from setuptools import setup, find_packages +"""Django/PostgreSQL implementation of the Meteor server.""" + +import os.path +import setuptools +import subprocess +from distutils import log +from distutils.version import StrictVersion +from distutils.command.build import build + +# setuptools 18.5 introduces support for the `platform_python_implementation` +# environment marker: https://github.com/jaraco/setuptools/pull/28 +__requires__ = 'setuptools>=18.5' + +assert StrictVersion(setuptools.__version__) >= StrictVersion('18.5'), \ + 'Installation from source requires setuptools>=18.5.' + + +class Build(build): + + """Build all files of a package.""" + + def run(self): + """Build our package.""" + cmdline = [ + 'meteor', + 'build', + '--directory', + '../build', + ] + meteor_dir = os.path.join( + os.path.dirname(__file__), + 'dddp', + 'test', + 'meteor_todos', + ) + log.info('Building meteor app %r (%s)', meteor_dir, ' '.join(cmdline)) + subprocess.check_call(cmdline, cwd=meteor_dir) + return build.run(self) + CLASSIFIERS = [ # Beta status until 1.0 is released @@ -40,51 +75,85 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", - "Framework :: Django :: 1.7", "Framework :: Django :: 1.8", + "Framework :: Django :: 1.9", ] -# Ensure correct dependencies between different python implementations. -IMPLEMENTATION_INSTALL_REQUIRES = { - # extra requirements for CPython implementation - 'CPython': [ - 'psycopg2>=2.5.4', - 'gevent>=1.1b6' if sys.version_info >= (3, 0) else 'gevent>=1.0', - ], - # extra requirements for all other Python implementations - None: [ - 'psycopg2cffi>=2.7.2', - 'gevent>=1.1b6', - ], -} - -setup( +setuptools.setup( name='django-ddp', - version='0.18.1', + version='0.19.0', description=__doc__, long_description=open('README.rst').read(), author='Tyson Clugg', author_email='tyson@clugg.net', - url='https://github.com/commoncode/django-ddp', + url='https://github.com/django-ddp/django-ddp', + keywords=[ + 'django ddp meteor websocket websockets realtime real-time live ' + 'liveupdate live-update livequery live-query' + ], license='MIT', - packages=find_packages(), - include_package_data=True, + packages=setuptools.find_packages(), + include_package_data=True, # install data files specified in MANIFEST.in + zip_safe=False, # TODO: Move dddp.test into it's own package. + setup_requires=[ + # packages required to run the setup script + __requires__, + ], install_requires=[ - 'Django>=1.7', - 'gevent-websocket>=0.9,!=0.9.4', + 'Django>=1.8', + 'django-dbarray>=0.2', 'meteor-ejson>=1.0', 'psycogreen>=1.0', - 'django-dbarray>=0.2', 'pybars3>=0.9.1', 'six>=1.10.0', - ] + IMPLEMENTATION_INSTALL_REQUIRES.get( - platform.python_implementation(), - IMPLEMENTATION_INSTALL_REQUIRES[None], # default to non-CPython reqs - ), + ], + extras_require={ + # We need gevent version dependent upon environment markers, but the + # extras_require seem to be a separate phase from setup/install of + # install_requires. So we specify gevent-websocket (which depends on + # gevent) here in order to honour environment markers. + '': [ + 'gevent-websocket>=0.9,!=0.9.4', + ], + # Django 1.9 doesn't support Python 3.3 + ':python_version=="3.3"': [ + 'Django<1.9', + ], + # CPython < 3.0 can use gevent 1.0 + ':platform_python_implementation=="CPython" and python_version<"3.0"': [ + 'gevent>=1.0', + ], + # everything else needs gevent 1.1 + ':platform_python_implementation!="CPython" or python_version>="3.0"': [ + 'gevent>=1.1rc2', + ], + # CPython can use plain old psycopg2 + ':platform_python_implementation=="CPython"': [ + 'psycopg2>=2.5.4', + ], + # everything else must use psycopg2cffi + ':platform_python_implementation != "CPython"': [ + 'psycopg2cffi>=2.7.2', + ], + 'develop': [ + # things you need to distribute a wheel from source (`make dist`) + 'Sphinx>=1.3.3', + 'Sphinx-PyPI-upload>=0.2.1', + 'twine>=1.6.4', + 'sphinxcontrib-dashbuilder>=0.1.0', + ], + }, entry_points={ 'console_scripts': [ 'dddp=dddp.main:main', ], }, classifiers=CLASSIFIERS, + test_suite='dddp.test.run_tests', + tests_require=[ + 'requests', + ], + cmdclass={ + 'build': Build, + }, ) diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index bc04b49..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r requirements.txt diff --git a/tox.ini b/tox.ini index 1788129..ae3a0ab 100644 --- a/tox.ini +++ b/tox.ini @@ -4,45 +4,158 @@ # and then run "tox" from this directory. [tox] -# require tox>=2.1.1 or refuse to run the tests. + +# require tox 2.1.1 or later minversion=2.1.1 -# return success even if some of the specified environments are missing +# don't fail if missing a python version specified in envlist skip_missing_interpreters=True -# "envlist" is a comma separated list of environments, each environment name -# contains factors separated by hyphens. For example, "py27-unittest" has 2 -# factors: "py27" and "unittest". Other settings such as "setenv" accept the -# factor names as a prefixes (eg: "unittest: ...") so that prefixed settings -# only apply if the environment being run contains that factor. - +# list of environments to run by default envlist = - py27-test, - py33-test, - py34-test, - py35-test, - pypy-test, + lint + clean + py33-django{1.8} + {py27,py34,py35,pypy,pypy3}-django{1.8,1.9} + report + [testenv] +# virtualenv only installs setuptools==0.18.2 but we need 0.18.5: +# - https://github.com/pypa/virtualenv/issues/807 +# - https://github.com/pypa/virtualenv/issues/801 +# - https://github.com/pypa/virtualenv/issues/717 +# - https://github.com/pypa/virtualenv/issues/781 +# - https://github.com/pypa/virtualenv/issues/580 +# - https://github.com/pypa/virtualenv/issues/563 +# - https://github.com/pypa/virtualenv/issues/491 +# wheel 0.25.0 needed for Python 3.5: +# - https://bitbucket.org/pypa/wheel/issues/146/wheel-building-fails-on-cpython-350b3 +install_command=sh -c 'pip install -U "setuptools>=18.5" "wheel>=0.25.0" "pip>=7.1.2" && pip install "$@" && sync' sh {opts} {packages} +whitelist_externals=sh + +# force clean environment each time recreate=True -usedevelop=True + +# build sdist from setup.py and install from that (validate setup.py) +usedevelop=False + +# list of environment variables passed through to commands passenv= - BUILD_NUMBER - BUILD_URL - XDG_CACHE_HOME + ; https://help.ubuntu.com/community/EnvironmentVariables#Other_environment_variables + USER + LOGNAME + HOME + TERM + TERMCAP -# stop running commands if previous commands fail -ignore_errors = False + ; https://help.ubuntu.com/community/EnvironmentVariables#Graphical_desktop-related_variables + DISPLAY + XDG_CACHE_HOME + C_INCLUDE_PATH + CFLAGS + ; https://wiki.jenkins-ci.org/display/JENKINS/Building+a+software+project + BUILD_NUMBER + BUILD_ID + BUILD_URL + NODE_NAME + JOB_NAME + BUILD_TAG + JENKINS_URL + EXECUTOR_NUMBER + JAVA_HOME + WORKSPACE + GIT_COMMIT + GIT_URL + GIT_BRANCH + + ; http://www.postgresql.org/docs/current/static/libpq-envars.html + PGHOST + PGHOSTADDR + PGPORT + PGDATABASE + PGUSER + PGPASSWORD + PGPASSFILE + PGSERVICE + PGSERVICEFILE + PGREALM + PGOPTIONS + PGAPPNAME + PGSSLMODE + PGREQUIRESSL + PGSSLCOMPRESSION + PGSSLCERT + PGSSLKEY + PGSSLROOTCERT + PGSSLCRL + PGREQUIREPEER + PGKRBSRVNAME + PGSSLLIB + PGCONNECT_TIMEOUT + PGCLIENTENCODING + PGDATESTYLE + PGTZ + PGGEQO + PGSYSCONFDIR + PGLOCALEDIR + +# `pip install -rrequierements.txt` <-- tox doesn't understand PEP-0496 Environment Markers. +# pypy coverage fails with --concurrency set to `gevent` +# pypy install gevent fails building wheel commands = - dist: check-manifest - py{27,33,34,35}-test: coverage run --concurrency=gevent {toxinidir}/dddp/test/manage.py test -v3 --noinput - pypy-test: coverage run {toxinidir}/dddp/test/manage.py test -v3 --noinput - test: coverage report - dist: {envpython} setup.py --quiet --no-user-cfg sdist --dist-dir={toxinidir}/dist/ - dist: {envpython} setup.py --quiet --no-user-cfg bdist_wheel --dist-dir={toxinidir}/dist/ + {py27,py33,py34,py35}: pip install -rrequirements.txt + {pypy,pypy3}: pip install --no-binary gevent -rrequirements.txt + {py27,py33,py34,py35}: coverage run --append --concurrency=gevent --source=dddp setup.py test + {pypy,pypy3}: coverage run --append --source=dddp setup.py test deps = - test: coverage - dist: check-manifest - dist: wheel + #-rrequirements.txt + django1.8: Django>=1.8,<1.9 + django1.9: Django>=1.9,<1.10 + coverage + + +[testenv:dist] +install_command=sh -c 'pip install -U "setuptools>=18.5" "wheel>=0.25.0" "pip>=7.1.2" && pip install "$@" && sync' sh {opts} {packages} + +whitelist_externals=sh + +commands = + check-manifest --ignore "dddp/test/build*,dddp/test/meteor_todos/.meteor/local*" + {envpython} setup.py --no-user-cfg sdist --dist-dir={toxinidir}/dist/ + {envpython} setup.py --no-user-cfg bdist_wheel --dist-dir={toxinidir}/dist/ + sh -c "cd docs && sphinx-build -b html -d _build/doctrees -D latex_paper_size=a4 . _build/html" + +usedevelop=True +deps = + -rrequirements.txt + -rrequirements-dev.txt + check-manifest + wheel + + +[testenv:clean] +skip_install=True +deps=coverage +commands= + coverage erase + + +[testenv:report] +skip_install=True +deps=coverage +commands= + coverage report + coverage html + + +[testenv:lint] +usedevelop=True +commands= + pip install -rrequirements.txt + prospector --doc-warnings --zero-exit {toxinidir}/dddp/ +deps = + prospector==0.10.2 + pylint==1.4.5