diff --git a/.gitignore b/.gitignore index 92c772f..c41cec0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,9 +23,9 @@ docs/node_modules/ .cache/ .coverage .tox/ -htmlcov/ -tests/report/ +/htmlcov/ +/report/ # meteor -dddp/test/build/ -dddp/test/meteor_todos/.meteor/ +test/build/ +test/meteor_todos/.meteor/ diff --git a/.travis.yml b/.travis.yml index 50fe7e8..e4116d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,25 +5,49 @@ sudo: false language: python +python: + - "2.7" + - "3.3" + - "3.4" + - "3.5" + - "pypy" + - "pypy3" env: - - TOXENV="py27-django1.8" - - TOXENV="py27-django1.9" - - TOXENV="py33-django1.8" - - TOXENV="py34-django1.8" - - TOXENV="py34-django1.9" - - TOXENV="py35-django1.8" - - TOXENV="py35-django1.9" - - TOXENV="pypy3-django1.8" - - TOXENV="pypy3-django1.9" - - TOXENV="pypy-django1.8" - - TOXENV="pypy-django1.9" + global: + - PGDATABASE="django_ddp_test_project" + - PGUSER="postgres" + matrix: + - DJANGO="1.8" + - DJANGO="1.9" + +# Django 1.9 dropped support for Python 3.3 +matrix: + exclude: + - python: "3.3" + env: DJANGO="1.9" + allow_failures: + - python: "3.3" + - python: "3.4" + - python: "3.5" + - python: "pypy" + - python: "pypy3" + +services: + - postgresql + +before_install: + - curl https://install.meteor.com/ | sh install: - - pip install tox coveralls + - pip install -U tox coveralls setuptools + +before_script: + - env | sort + - psql -c "create database ${PGDATABASE};" postgres script: - - tox + - PATH="$HOME/.meteor:$PATH" tox -vvvv -e $( echo $TRAVIS_PYTHON_VERSION | sed -e 's/^2\./py2/' -e 's/^3\./py3/' )-django${DJANGO} after_success: coveralls diff --git a/.travis.yml.ok b/.travis.yml.ok new file mode 100644 index 0000000..ef67091 --- /dev/null +++ b/.travis.yml.ok @@ -0,0 +1,3 @@ +1.8.0 +180e6379f9c19f2fc577e42388d93b4590c6f37d .travis.yml +valid diff --git a/.travis.yml.sh b/.travis.yml.sh deleted file mode 100755 index 89fde3d..0000000 --- a/.travis.yml.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -cat< fast boot (1-6s) -# https://docs.travis-ci.com/user/ci-environment/ -sudo: false - -language: python - -env: -$( tox -l | grep '^py' | sort -n | sed -e 's/^.*$/ - TOXENV="\0"/' ) - -install: - - pip install tox coveralls - -script: - - tox - -after_success: - coveralls -EOF diff --git a/CHANGES.rst b/CHANGES.rst index 04ef82f..d847589 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,8 +4,24 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to `Semantic Versioning `_. -0.19.1 ------- +0.19.1 (2016-01-28) +------------------- +* Consistent handling of client errors (`MeteorError`) which shouldn't + be logged. +* Reduce wheel size from 204KB to 68KB by removing dddp.test package. +* Reduce sdist size from 10MB to 209KB by removing Meteor build from + test suite. +* Improve test suite with coverage now 65% when tested via Travis CI. +* Dropped support for Python 3.3. +* Fix for #3 -- drop support for Django 1.7, add support for Django 1.9 + - thanks @schinckel. +* Re-raise exceptions from DDP WebSocket handlers rather than swallowing + them. +* Fix for #33 -- Add `meteor_autoupdate_clientVersions` publication. + + +0.19.0 (2015-12-16) +------------------- * Dropped support for Django 1.7 (support expired on December 1 2015, see https://www.djangoproject.com/download/#supported-versions). * Require `setuptools>=18.5` at install time due to use of diff --git a/MANIFEST.in b/MANIFEST.in index 377c556..e3799b7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,16 +1,18 @@ -include LICENSE include *.rst include *.sh include *.txt -include requirements*.txt include .gitignore +include LICENSE +include requirements*.txt include Makefile -exclude tox.ini -graft dddp/test/meteor_todos -prune dddp/test/build -prune dddp/test/meteor_todos/.meteor/local graft docs +graft tests prune docs/_build prune docs/node_modules -exclude .travis.yml.sh +prune tests/build +prune tests/meteor_todos/.meteor/local +prune */__pycache__ +prune *.pyc +exclude .travis.yml.ok exclude .travis.yml +exclude tox.ini diff --git a/Makefile b/Makefile index 91e6e1b..67009ce 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,10 @@ WHEEL := dist/$(subst -,_,${NAME})-${VERSION}-py2.py3-none-any.whl .INTERMEDIATE: dist.intermediate docs -all: .travis.yml docs dist +all: .travis.yml.ok docs dist test: - tox -vvv + tox --skip-missing-interpreters -vvv clean: clean-docs clean-dist clean-pyc @@ -19,7 +19,7 @@ clean-docs: $(MAKE) -C docs/ clean clean-dist: - rm -rf "${SDIST}" "${WHEEL}" dddp/test/build/ dddp/test/meteor_todos/.meteor/local/ + rm -rf "${SDIST}" "${WHEEL}" tests/build/ tests/meteor_todos/.meteor/local/ clean-pyc: find . -type f -name \*.pyc -print0 | xargs -0 rm -f @@ -33,11 +33,11 @@ dist: ${SDIST} ${WHEEL} ${SDIST}: dist.intermediate @echo "Testing ${SDIST}..." - tox --notest --installpkg ${SDIST} + tox --skip-missing-interpreters --notest --installpkg ${SDIST} ${WHEEL}: dist.intermediate @echo "Testing ${WHEEL}..." - tox --notest --installpkg ${WHEEL} + tox --skip-missing-interpreters --notest --installpkg ${WHEEL} dist.intermediate: $(shell find dddp -type f) tox -e dist @@ -50,5 +50,7 @@ upload-pypi: ${SDIST} ${WHEEL} upload-docs: docs/_build/ python setup.py upload_sphinx --upload-dir="$ "$@" +.travis.yml.ok: .travis.yml + @travis --version > "$@" || { echo 'Install travis command line client?'; exit 1; } + sha1sum "$<" >> "$@" + travis lint --exit-code | tee -a "$@" diff --git a/README.rst b/README.rst index 58e64ab..34fc589 100644 --- a/README.rst +++ b/README.rst @@ -2,13 +2,20 @@ Django DDP ========== -`Django DDP`_ is a Django_/PostgreSQL_ implementation of the Meteor DDP server, allowing Meteor_ to subscribe to changes on Django_ models. Released under the MIT license. +`Django DDP`_ is a Django_/PostgreSQL_ implementation of the Meteor DDP +server, allowing Meteor_ to subscribe to changes on Django_ models. +Released under the MIT license. Requirements ------------ -You must be using PostgreSQL_ with psycopg2_ in your Django_ project for django-ddp to work. There is no requirement on any asynchronous framework such as Reddis or crossbar.io as they are simply not needed given the asynchronous support provided by PostgreSQL_ with psycopg2_. +You must be using PostgreSQL_ with psycopg2_ in your Django_ project +for django-ddp to work. There is no requirement on any asynchronous +framework such as Redis or crossbar.io as they are simply not needed +given the asynchronous support provided by PostgreSQL_ with psycopg2_. +Since the test suite includes an example Meteor_ project, running that +requires that Meteor_ is installed (and `meteor` is in your `PATH`). Installation ------------ @@ -19,7 +26,8 @@ Install the latest release from pypi (recommended): pip install django-ddp -Clone and use development version direct from GitHub to test pre-release code (no GitHub account required): +Clone and use development version direct from GitHub to test pre-release +code (no GitHub account required): .. code:: sh @@ -43,9 +51,15 @@ Overview and getting started Scalability ----------- -All database queries to support DDP events are done once by the server instance that has made changes via the Django ORM. Django DDP multiplexes messages for active subscriptions, broadcasting an aggregated change message on channels specific to each Django model that has been published. +All database queries to support DDP events are done once by the server +instance that has made changes via the Django ORM. Django DDP multiplexes +messages for active subscriptions, broadcasting an aggregated change +message on channels specific to each Django model that has been published. -Peer servers subscribe to aggregate broadcast events which are de-multiplexed and dispatched to individual client connections. No additional database queries are required for de-multiplexing or dispatch by peer servers. +Peer servers subscribe to aggregate broadcast events which are +de-multiplexed and dispatched to individual client connections. +No additional database queries are required for de-multiplexing +or dispatch by peer servers. Limitations @@ -138,7 +152,7 @@ Start the Django DDP service: Using django-ddp as a secondary DDP connection (RAPID DEVELOPMENT) ------------------------------------------------------------------ -Running in this manner allows rapid development through use of the hot +Running in this manner allows rapid development through use of the hot code push features provided by Meteor. Connect your Meteor application to the Django DDP service: @@ -165,13 +179,13 @@ Start Meteor (from within your meteor application directory): Using django-ddp as the primary DDP connection (RECOMMENDED) ------------------------------------------------------------ -If you'd prefer to not have two DDP connections (one to Meteor and one -to django-ddp) you can set the `DDP_DEFAULT_CONNECTION_URL` environment -variable to use the specified URL as the primary DDP connection in -Meteor. When doing this, you won't need to use `DDP.connect(...)` or -specify `{connection: Django}` on your collections. Running with -django-ddp as the primary connection is recommended, and indeed required -if you wish to use `dddp.accounts` to provide authentication using +If you'd prefer to not have two DDP connections (one to Meteor and one +to django-ddp) you can set the `DDP_DEFAULT_CONNECTION_URL` environment +variable to use the specified URL as the primary DDP connection in +Meteor. When doing this, you won't need to use `DDP.connect(...)` or +specify `{connection: Django}` on your collections. Running with +django-ddp as the primary connection is recommended, and indeed required +if you wish to use `dddp.accounts` to provide authentication using `django.contrib.auth` to your meteor app. .. code:: sh @@ -182,7 +196,7 @@ if you wish to use `dddp.accounts` to provide authentication using Serving your Meteor applications from django-ddp ------------------------------------------------ -First, you will need to build your meteor app into a directory (examples +First, you will need to build your meteor app into a directory (examples below assume target directory named `myapp`): .. code:: sh @@ -257,7 +271,7 @@ Contributors `Muhammed Thanish `_ * Making the `DDP Test Suite `_ available. -This project is forever grateful for the love, support and respect given +This project is forever grateful for the love, support and respect given by the awesome team at `Common Code`_. .. _Django DDP: https://github.com/django-ddp/django-ddp diff --git a/dddp/__init__.py b/dddp/__init__.py index e027d7f..9458005 100644 --- a/dddp/__init__.py +++ b/dddp/__init__.py @@ -4,7 +4,7 @@ import sys from gevent.local import local from dddp import alea -__version__ = '0.19.0' +__version__ = '0.19.1' __url__ = 'https://github.com/django-ddp/django-ddp' default_app_config = 'dddp.apps.DjangoDDPConfig' @@ -42,6 +42,45 @@ def greenify(): patch_psycopg() +class MeteorError(Exception): + + """ + MeteorError. + + This exception can be thrown by DDP API endpoints (methods) and publication + methods. MeteorError is not expected to be logged or shown by the server, + leaving all handling of the error condition for the client. + + Args: + error (str): A string code uniquely identifying this kind of error. + This string should be used by clients to determine the appropriate + action to take, instead of attempting to parse the reason or detail + fields. + reason (Optional[str]): A short human-readable summary of the error. + detail (Optional[str]): Additional information about the error. + When returning errors to clients, Django DDP will default this to a + textual stack trace if `django.conf.settings.DEBUG` is `True`. + """ + + def __init__(self, error, reason=None, details=None, **kwargs): + """MeteorError constructor.""" + super(MeteorError, self).__init__(error, reason, details, kwargs) + + def as_dict(self, **kwargs): + """Return an error dict for self.args and kwargs.""" + error, reason, details, err_kwargs = self.args + result = { + key: val + for key, val in { + 'error': error, 'reason': reason, 'details': details, + }.items() + if val is not None + } + result.update(err_kwargs) + result.update(kwargs) + return result + + class AlreadyRegistered(Exception): """Raised when registering over the top of an existing registration.""" @@ -125,7 +164,7 @@ THREAD_LOCAL_FACTORIES = { 'user_ddp_id': lambda: None, 'user': lambda: None, } -THREAD_LOCAL = ThreadLocal() +THREAD_LOCAL = this = ThreadLocal() # pylint: disable=invalid-name METEOR_ID_CHARS = u'23456789ABCDEFGHJKLMNPQRSTWXYZabcdefghijkmnopqrstuvwxyz' diff --git a/dddp/accounts/ddp.py b/dddp/accounts/ddp.py index 8896f1c..b8c4abd 100644 --- a/dddp/accounts/ddp.py +++ b/dddp/accounts/ddp.py @@ -21,16 +21,16 @@ from django.dispatch import Signal from django.utils import timezone from dddp import ( - THREAD_LOCAL_FACTORIES, THREAD_LOCAL as this, ADDED, REMOVED, + THREAD_LOCAL_FACTORIES, this, MeteorError, + ADDED, REMOVED, meteor_random_id, ) from dddp.models import get_meteor_id, get_object, Subscription from dddp.api import API, APIMixin, api_endpoint, Collection, Publication -from dddp.websocket import MeteorError -# pylint dones't like lower case attribute names on modules, but it's the normal -# thing to do for Django signal names. --> pylint: disable=C0103 +# pylint doesn't like lower case attribute names on modules, but it's the +# normal thing to do for Django signal names. --> pylint: disable=C0103 create_user = Signal(providing_args=['request', 'params']) password_changed = Signal(providing_args=['request', 'user']) forgot_password = Signal(providing_args=['request', 'user', 'token', 'expiry']) @@ -49,7 +49,7 @@ HASH_MINUTES_VALID = { HashPurpose.PASSWORD_RESET: int( getattr( # keep possible attack window short to reduce chance of account - # takeover through later discovery of password reset email message. + # takeover through later discovery of password reset email message. settings, 'DDP_PASSWORD_RESET_MINUTES_VALID', '1440', # 24 hours ) ), @@ -67,8 +67,8 @@ def iter_auth_hashes(user, purpose, minutes_valid): """ Generate auth tokens tied to user and specified purpose. - The hash expires at midnight on the minute of now + minutes_valid, such that - when minutes_valid=1 you get *at least* 1 minute to use the token. + The hash expires at midnight on the minute of now + minutes_valid, such + that when minutes_valid=1 you get *at least* 1 minute to use the token. """ now = timezone.now().replace(microsecond=0, second=0) for minute in range(minutes_valid + 1): @@ -190,7 +190,7 @@ class Users(Collection): if key == prefixed('name'): result['full_name'] = val else: - raise ValueError('Bad profile key: %r' % key) + raise MeteorError(400, 'Bad profile key: %r' % key) return result @api_endpoint @@ -278,7 +278,9 @@ class Auth(APIMixin): for col_post, query in post.items(): try: qs_pre = pre[col_post] - query = query.exclude(pk__in=qs_pre.order_by().values('pk')) + query = query.exclude( + pk__in=qs_pre.order_by().values('pk'), + ) except KeyError: # collection not included pre-auth, everything is added. pass @@ -377,25 +379,26 @@ class Auth(APIMixin): return password else: # Meteor is trying to be smart by doing client side hashing of the - # password so that passwords are "...not sent in plain text over the - # wire". This behaviour doesn't make HTTP any more secure - it just - # gives a false sense of security as replay attacks and + # password so that passwords are "...not sent in plain text over + # the wire". This behaviour doesn't make HTTP any more secure - + # it just gives a false sense of security as replay attacks and # code-injection are both still viable attack vectors for the # malicious MITM. Also as no salt is used with hashing, the # passwords are vulnerable to rainbow-table lookups anyway. # - # If you're doing security, do it right from the very outset. Fors + # If you're doing security, do it right from the very outset. For # web services that means using SSL and not relying on half-baked # security concepts put together by people with no security # background. # - # We protest loudly to anyone who cares to listen in the server logs - # until upstream developers see the light and drop the password - # hashing mis-feature. + # We protest loudly to anyone who cares to listen in the server + # logs until upstream developers see the light and drop the + # password hashing mis-feature. raise MeteorError( - 400, - "Outmoded password hashing, run " - "`meteor add tysonclugg:accounts-secure` to fix.", + 426, + "Outmoded password hashing: " + "https://github.com/meteor/meteor/issues/4363", + upgrade='meteor add tysonclugg:accounts-secure', ) @api_endpoint('createUser') @@ -407,7 +410,9 @@ class Auth(APIMixin): params=params, ) if len(receivers) == 0: - raise MeteorError(501, 'Handler for `create_user` not registered.') + raise NotImplementedError( + 'Handler for `create_user` not registered.' + ) user = receivers[0][1] user = auth.authenticate( username=user.get_username(), password=params['password'], @@ -475,7 +480,7 @@ class Auth(APIMixin): minutes_valid=HASH_MINUTES_VALID[HashPurpose.RESUME_LOGIN], ) - # Call to `authenticate` was unable to verify the username and password. + # Call to `authenticate` couldn't verify the username and password. # It will have sent the `user_login_failed` signal, no need to pass the # `username` argument to auth_failed(). self.auth_failed() diff --git a/dddp/accounts/tests.py b/dddp/accounts/tests.py new file mode 100644 index 0000000..04f53cc --- /dev/null +++ b/dddp/accounts/tests.py @@ -0,0 +1,93 @@ +"""Django DDP Accounts test suite.""" +from __future__ import unicode_literals + +import sys +from dddp import tests +from django.contrib.auth import get_user_model + + +# gevent-websocket doesn't work with Python 3 yet +@tests.expected_failure_if(sys.version_info.major == 3) +class AccountsTestCase(tests.DDPServerTestCase): + + def test_login_no_accounts(self): + sockjs = self.server.sockjs('/sockjs/1/a/websocket') + + resp = sockjs.websocket.recv() + self.assertEqual(resp, 'o') + + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'server_id': '0'}, + ], + ) + + sockjs.connect('1', 'pre2', 'pre1') + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'msg': 'connected', 'session': msgs[0]['session']}, + ], + ) + + id_ = sockjs.call( + 'login', {'user': 'invalid@example.com', 'password': 'foo'}, + ) + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + { + 'msg': 'result', 'id': id_, + 'error': { + 'error': 403, 'reason': 'Authentication failed.', + }, + }, + ], + ) + + sockjs.close() + + def test_login_new_account(self): + User = get_user_model() + new_user = User.objects.create_user( + 'user@example.com', 's3cre7-pa55w0rd!', + ) + sockjs = self.server.sockjs('/sockjs/1/a/websocket') + + resp = sockjs.websocket.recv() + self.assertEqual(resp, 'o') + + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'server_id': '0'}, + ], + ) + + sockjs.connect('1', 'pre2', 'pre1') + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'msg': 'connected', 'session': msgs[0]['session']}, + ], + ) + + id_ = sockjs.call( + 'login', { + 'user': 'user@example.com', 'password': 's3cre7-pa55w0rd!', + }, + ) + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + { + 'msg': 'result', 'id': id_, + 'error': { + 'error': 403, 'reason': 'Authentication failed.', + }, + }, + ], + ) + + sockjs.close() diff --git a/dddp/alea.py b/dddp/alea.py old mode 100755 new mode 100644 index b8a91ee..4b0bd42 --- a/dddp/alea.py +++ b/dddp/alea.py @@ -160,8 +160,3 @@ class Alea(object): def hex_string(self, digits): """Return a hex string of `digits` length.""" return self.random_string(digits, '0123456789abcdef') - - -if __name__ == '__main__': - import doctest - doctest.testmod() diff --git a/dddp/api.py b/dddp/api.py index 0e77f47..e3eac98 100644 --- a/dddp/api.py +++ b/dddp/api.py @@ -4,21 +4,19 @@ from __future__ import absolute_import, unicode_literals, print_function # standard library import collections from copy import deepcopy -import traceback +import inspect import uuid # requirements -import dbarray from django.conf import settings import django.contrib.postgres.fields from django.db import connections, router, transaction -from django.db.models import aggregates, Q +from django.db.models import Q try: # pylint: disable=E0611 from django.db.models.expressions import ExpressionNode except ImportError: from django.db.models import Expression as ExpressionNode -from django.db.models.sql import aggregates as sql_aggregates from django.utils.encoding import force_text from django.utils.module_loading import import_string from django.db import DatabaseError @@ -27,9 +25,7 @@ import ejson import six # django-ddp -from dddp import ( - AlreadyRegistered, THREAD_LOCAL as this, ADDED, CHANGED, REMOVED, -) +from dddp import AlreadyRegistered, this, ADDED, CHANGED, REMOVED, MeteorError from dddp.models import ( AleaIdField, Connection, Subscription, get_meteor_id, get_meteor_ids, ) @@ -43,55 +39,16 @@ API_ENDPOINT_DECORATORS = [ XMIN = {'select': {'xmin': "'xmin'"}} +# Only do this if < django1.9? -class Sql(object): +if django.VERSION < (1, 9): + from django.db.models import aggregates - """Extensions to django.db.models.sql.aggregates module.""" - - class Array(sql_aggregates.Aggregate): - - """Array SQL aggregate extension.""" - - lookup_name = 'array' - sql_function = 'array_agg' - -sql_aggregates.Array = Sql.Array - - -# pylint: disable=W0223 -class Array(aggregates.Aggregate): - - """Array aggregate function.""" - - func = 'ARRAY' - function = 'array_agg' - name = 'Array' - - def add_to_query(self, query, alias, col, source, is_summary): - """Override source field internal type so the raw array is returned.""" - @six.add_metaclass(dbarray.ArrayFieldMetaclass) - class ArrayField(dbarray.ArrayFieldBase, source.__class__): - - """ArrayField for override.""" - - @staticmethod - def get_internal_type(): - """Return ficticious type so Django doesn't cast as int.""" - return 'ArrayType' - - new_source = ArrayField() - try: - super(Array, self).add_to_query( - query, alias, col, new_source, is_summary, - ) - except AttributeError: - query.aggregates[alias] = new_source - - def convert_value(self, value, expression, connection, context): - """Convert value from format returned by DB driver to Python value.""" - if not value: - return [] - return value + # pylint: disable=W0223 + class ArrayAgg(aggregates.Aggregate): + function = 'ARRAY_AGG' +else: + from django.contrib.postgres.aggregates import ArrayAgg def api_endpoint(path_or_func=None, decorate=True): @@ -165,7 +122,7 @@ class APIMeta(type): """DDP API metaclass.""" - def __new__(mcs, name, bases, attrs): + def __new__(cls, name, bases, attrs): """Create a new APIMixin class.""" attrs['name'] = attrs.pop('name', None) or name name_format = attrs.get('name_format', None) @@ -176,7 +133,7 @@ class APIMeta(type): pass elif api_path_prefix_format is not None: attrs['api_path_prefix'] = api_path_prefix_format.format(**attrs) - return super(APIMeta, mcs).__new__(mcs, name, bases, attrs) + return super(APIMeta, cls).__new__(cls, name, bases, attrs) class APIMixin(object): @@ -329,7 +286,7 @@ class Collection(APIMixin): if isinstance(user_rels, basestring): user_rels = [user_rels] user_rel_map = { - '_user_rel_%d' % index: Array(user_rel) + '_user_rel_%d' % index: ArrayAgg(user_rel) for index, user_rel in enumerate(user_rels) } @@ -677,19 +634,7 @@ class DDP(APIMixin): @api_endpoint def sub(self, id_, name, *params): """Create subscription, send matched objects that haven't been sent.""" - try: - return self.do_sub(id_, name, False, *params) - except Exception as err: - this.send({ - 'msg': 'nosub', - 'id': id_, - 'error': { - 'error': 500, - 'errorType': 'Meteor.Error', - 'message': '%s' % err, - 'reason': 'Subscription failed', - }, - }) + return self.do_sub(id_, name, False, *params) @transaction.atomic def do_sub(self, id_, name, silent, *params): @@ -698,16 +643,7 @@ class DDP(APIMixin): pub = self.get_pub_by_name(name) except KeyError: if not silent: - this.send({ - 'msg': 'nosub', - 'id': id_, - 'error': { - 'error': 404, - 'errorType': 'Meteor.Error', - 'message': 'Subscription not found [404]', - 'reason': 'Subscription not found', - }, - }) + raise MeteorError(404, 'Subscription not found') return sub, created = Subscription.objects.get_or_create( connection_id=this.ws.connection.pk, @@ -788,41 +724,16 @@ class DDP(APIMixin): try: handler = self.api_path_map()[method] except KeyError: - print('Unknown method: %s %r' % (method, params)) - this.send({ - 'msg': 'result', - 'id': id_, - 'error': { - 'error': 404, - 'errorType': 'Meteor.Error', - 'message': 'Unknown method: %s %r' % (method, params), - 'reason': 'Method not found', - }, - }) - return - params_repr = repr(params) + raise MeteorError(404, 'Method not found', method) try: - result = handler(*params) - msg = {'msg': 'result', 'id': id_} - if result is not None: - msg['result'] = result - this.send(msg) - except Exception as err: # log err+stack trace -> pylint: disable=W0703 - details = traceback.format_exc() - print(id_, method, params_repr) - print(details) - this.ws.logger.error(err, exc_info=True) - msg = { - 'msg': 'result', - 'id': id_, - 'error': { - 'error': 500, - 'reason': str(err), - }, - } - if settings.DEBUG: - msg['error']['details'] = details - this.send(msg) + inspect.getcallargs(handler, *params) + except TypeError as err: + raise MeteorError(400, '%s' % err) + result = handler(*params) + msg = {'msg': 'result', 'id': id_} + if result is not None: + msg['result'] = result + this.send(msg) def register(self, api_or_iterable): """Register an API endpoint.""" diff --git a/dddp/apps.py b/dddp/apps.py index fe0090b..e233d23 100644 --- a/dddp/apps.py +++ b/dddp/apps.py @@ -21,7 +21,10 @@ class DjangoDDPConfig(AppConfig): raise ImproperlyConfigured('No databases configured.') for (alias, conf) in settings.DATABASES.items(): engine = conf['ENGINE'] - if engine != 'django.db.backends.postgresql_psycopg2': + if engine not in [ + 'django.db.backends.postgresql', + 'django.db.backends.postgresql_psycopg2', + ]: warnings.warn( 'Database %r uses unsupported %r engine.' % ( alias, engine, diff --git a/dddp/ddp.py b/dddp/ddp.py index da7d54d..6583844 100644 --- a/dddp/ddp.py +++ b/dddp/ddp.py @@ -1,7 +1,15 @@ -from dddp import THREAD_LOCAL as this +from django.contrib import auth +from dddp import THREAD_LOCAL from dddp.api import API, Publication from dddp.logging import LOGS_NAME -from django.contrib import auth + + +class ClientVersions(Publication): + """Publication for `meteor_autoupdate_clientVersions`.""" + + name = 'meteor_autoupdate_clientVersions' + + queries = [] class Logs(Publication): @@ -10,7 +18,7 @@ class Logs(Publication): users = auth.get_user_model() def get_queries(self): - user_pk = getattr(this, 'user_id', False) + user_pk = getattr(THREAD_LOCAL, 'user_id', False) if user_pk: if self.users.objects.filter( pk=user_pk, @@ -21,4 +29,4 @@ class Logs(Publication): raise ValueError('User not permitted.') -API.register([Logs]) +API.register([ClientVersions, Logs]) diff --git a/dddp/postgres.py b/dddp/postgres.py index ef8718a..419b103 100644 --- a/dddp/postgres.py +++ b/dddp/postgres.py @@ -77,7 +77,7 @@ class PostgresGreenlet(gevent.Greenlet): gevent.select.select, [conn], [], [], timeout=None, ) - self.select_greenlet.join() + self.select_greenlet.get() except gevent.GreenletExit: self._stop_event.set() finally: @@ -93,6 +93,8 @@ class PostgresGreenlet(gevent.Greenlet): self._stop_event.set() if self.select_greenlet is not None: self.select_greenlet.kill() + self.select_greenlet.get() + gevent.sleep() def poll(self, conn): """Poll DB socket and process async tasks.""" diff --git a/dddp/test/__init__.py b/dddp/test/__init__.py deleted file mode 100644 index 8740a3b..0000000 --- a/dddp/test/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# This file mainly exists to allow `python setup.py test` to work. -import os -import sys - -import dddp -import django -from django.test.utils import get_runner -from django.conf import settings - - -def run_tests(): - os.environ['DJANGO_SETTINGS_MODULE'] = 'dddp.test.test_project.settings' - dddp.greenify() - django.setup() - test_runner = get_runner(settings)() - failures = test_runner.run_tests(['dddp', 'dddp.test.django_todos']) - sys.exit(bool(failures)) diff --git a/dddp/test/django_todos/views.py b/dddp/test/django_todos/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/dddp/test/django_todos/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/dddp/test/manage.py b/dddp/test/manage.py deleted file mode 100755 index d18fe6d..0000000 --- a/dddp/test/manage.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -"""Entry point for Django DDP test project.""" -import os -import sys - - -if __name__ == "__main__": - os.environ['DJANGO_SETTINGS_MODULE'] = 'dddp.test.test_project.settings' - - from dddp import greenify - greenify() - - from django.core.management import execute_from_command_line - - execute_from_command_line(sys.argv) diff --git a/dddp/tests.py b/dddp/tests.py index 3f98d06..3286ae1 100644 --- a/dddp/tests.py +++ b/dddp/tests.py @@ -1,34 +1,146 @@ """Django DDP test suite.""" -from __future__ import unicode_literals +from __future__ import absolute_import, unicode_literals import doctest import errno import os import socket +import sys import unittest -from django.test import TestCase +import django.test +import ejson +import gevent +import dddp import dddp.alea from dddp.main import DDPLauncher # pylint: disable=E0611, F0401 from six.moves.urllib_parse import urljoin -os.environ['DJANGO_SETTINGS_MODULE'] = 'dddp.test.test_project.settings' +os.environ['DJANGO_SETTINGS_MODULE'] = 'test_project.settings' DOCTEST_MODULES = [ dddp.alea, ] -class DDPServerTestCase(TestCase): +def expected_failure_if(condition): + """Decorator to conditionally wrap func in unittest.expectedFailure.""" + if callable(condition): + condition = condition() + if condition: + # condition is True, expect failure. + return unittest.expectedFailure + else: + # condition is False, expect success. + return lambda func: func - """Test case that starts a DDP server.""" + +class WebSocketClient(object): + + """WebSocket client.""" + + # WEBSOCKET + + def __init__(self, *args, **kwargs): + """Create WebSocket connection to URL.""" + import websocket + self.websocket = websocket.create_connection(*args, **kwargs) + self.call_seq = 0 + self._prng = dddp.RandomStreams() + + def set_seed(self, seed): + """Set PRNG seed value.""" + self._prng.random_seed = seed + + def send(self, **msg): + """Send message.""" + self.websocket.send(ejson.dumps(msg)) + + def recv(self): + """Receive a message.""" + raw = self.websocket.recv() + return ejson.loads(raw) + + def close(self): + """Close the connection.""" + self.websocket.close() + + # CONTEXT MANAGER + + def __enter__(self): + """Enter context block.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Exit context block, close connection.""" + self.websocket.close() + + # Alea PRNG (seeded) + def meteor_random_id(self, name=None, length=17): + """Return seeded PRNG.""" + return self._prng[name].random_string(length, dddp.METEOR_ID_CHARS) + + # DDP + def next_id(self): + """Return next `id` from sequence.""" + self.call_seq += 1 + return self.call_seq + + def connect(self, *versions): + """Connect with given versions.""" + self.send(msg='connect', version=versions[0], support=versions) + + def ping(self, id_=None): + """Ping with optional id.""" + if id_: + self.send(msg='ping', id=id_) + else: + self.send(msg='ping') + + def call(self, method, *args): + """Make a method call.""" + id_ = self.next_id() + self.send(msg='method', method=method, params=args, id=id_) + return id_ + + def sub(self, name, *params): + """Subscribe to a named publication.""" + sub_id = self.meteor_random_id() + self.send(msg='sub', id=sub_id, name=name, params=params) + return sub_id + + def unsub(self, sub_id): + """Unsubscribe from a publication by sub_id.""" + self.send(msg='unsub', id=sub_id) + + +class SockJSClient(WebSocketClient): + + """SockJS wrapped WebSocketClient.""" + + def send(self, **msg): + """Send a SockJS wrapped msg.""" + self.websocket.send(ejson.dumps([ejson.dumps(msg)])) + + def recv(self): + """Receive a SockJS wrapped msg.""" + raw = self.websocket.recv() + if not raw.startswith('a'): + raise ValueError('Invalid response: %r' % raw) + wrapped = ejson.loads(raw[1:]) + return [ejson.loads(msg) for msg in wrapped] + + +class DDPTestServer(object): + + """DDP server with auto start and stop.""" server_addr = '127.0.0.1' server_port_range = range(8000, 8080) ssl_certfile_path = None ssl_keyfile_path = None - def setUp(self): + def __init__(self): """Fire up the DDP server.""" self.server_port = 8000 kwargs = {} @@ -55,10 +167,22 @@ class DDPServerTestCase(TestCase): continue # port in use, try next port. raise RuntimeError('Failed to start DDP server.') - def tearDown(self): + def stop(self): """Shut down the DDP server.""" self.server.stop() + def websocket(self, url, *args, **kwargs): + """Return a WebSocketClient for the given URL.""" + return WebSocketClient( + self.url(url).replace('http', 'ws'), *args, **kwargs + ) + + def sockjs(self, url, *args, **kwargs): + """Return a SockJSClient for the given URL.""" + return SockJSClient( + self.url(url).replace('http', 'ws'), *args, **kwargs + ) + def url(self, path): """Return full URL for given path.""" return urljoin( @@ -67,9 +191,48 @@ class DDPServerTestCase(TestCase): ) -class LaunchTestCase(DDPServerTestCase): +class DDPServerTestCase(django.test.TransactionTestCase): - """Test that server launches and handles GET request.""" + _server = None + + @property + def server(self): + if self._server is None: + self._server = DDPTestServer() + return self._server + + def tearDown(self): + """Stop the DDP server, and reliably close any open DB transactions.""" + if self._server is not None: + self._server.stop() + self._server = None + # OK, let me explain what I think is going on here... + # 1. Tests run, but cursors aren't closed. + # 2. Open cursors keeping queries running on DB. + # 3. Django TestCase.tearDown() tries to `sqlflush` + # 4. DB complains that queries from (2) are still running. + # Solution is to use the open DB connection and execute a DB noop query + # such as `SELECT pg_sleep(0)` to force another round trip to the DB. + # If you have another idea as to what is going on, submit an issue. :) + from django.db import connection, close_old_connections + close_old_connections() + cur = connection.cursor() + cur.execute('SELECT pg_sleep(0);') + cur.fetchall() + cur.close() + connection.close() + gevent.sleep() + super(DDPServerTestCase, self).tearDown() + + def url(self, path): + return self.server.url(path) + + +# gevent-websocket doesn't work with Python 3 yet +@expected_failure_if(sys.version_info.major == 3) +class HttpTestCase(DDPServerTestCase): + + """Test that server launches and handles HTTP requests.""" def test_get(self): """Perform HTTP GET.""" @@ -78,6 +241,164 @@ class LaunchTestCase(DDPServerTestCase): self.assertEqual(resp.status_code, 200) +# gevent-websocket doesn't work with Python 3 yet +@expected_failure_if(sys.version_info.major == 3) +class WebSocketTestCase(DDPServerTestCase): + + """Test that server launches and handles WebSocket connections.""" + + def test_sockjs_connect_ping(self): + """SockJS connect.""" + sockjs = self.server.sockjs('/sockjs/1/a/websocket') + + resp = sockjs.websocket.recv() + self.assertEqual(resp, 'o') + + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'server_id': '0'}, + ], + ) + + sockjs.connect('1', 'pre2', 'pre1') + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'msg': 'connected', 'session': msgs[0].get('session', None)}, + ], + ) + + # first without `id` + sockjs.ping() + msgs = sockjs.recv() + self.assertEqual(msgs, [{'msg': 'pong'}]) + + # then with `id` + id_ = sockjs.next_id() + sockjs.ping(id_) + msgs = sockjs.recv() + self.assertEqual(msgs, [{'msg': 'pong', 'id': id_}]) + + sockjs.close() + + def test_sockjs_connect_sub_unsub(self): + """SockJS connect.""" + sockjs = self.server.sockjs('/sockjs/1/a/websocket') + + resp = sockjs.websocket.recv() + self.assertEqual(resp, 'o') + + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'server_id': '0'}, + ], + ) + + sockjs.connect('1', 'pre2', 'pre1') + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'msg': 'connected', 'session': msgs[0].get('session', None)}, + ], + ) + + # subscribe to `meteor_autoupdate_clientVersions` publication + sub_id = sockjs.sub('meteor_autoupdate_clientVersions') + msgs = sockjs.recv() + self.assertEqual(msgs, [{'msg': 'ready', 'subs': [sub_id]}]) + + # unsubscribe from publication + sockjs.unsub(sub_id) + msgs = sockjs.recv() + self.assertEqual(msgs, [{'msg': 'nosub', 'id': sub_id}]) + + sockjs.close() + + def test_call_missing_arguments(self): + """Connect and login without any arguments.""" + sockjs = self.server.sockjs('/sockjs/1/a/websocket') + + resp = sockjs.websocket.recv() + self.assertEqual(resp, 'o') + + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'server_id': '0'}, + ], + ) + + sockjs.connect('1', 'pre2', 'pre1') + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'msg': 'connected', 'session': msgs[0].get('session', None)}, + ], + ) + + id_ = sockjs.call('login') # expects `credentials` argument + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + { + 'msg': 'result', + 'error': { + 'error': 400, + 'reason': + 'login() takes exactly 2 arguments (1 given)', + }, + 'id': id_, + }, + ], + ) + + sockjs.close() + + def test_call_extra_arguments(self): + """Connect and login with extra arguments.""" + with self.server.sockjs('/sockjs/1/a/websocket') as sockjs: + + resp = sockjs.websocket.recv() + self.assertEqual(resp, 'o') + + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + {'server_id': '0'}, + ], + ) + + sockjs.connect('1', 'pre2', 'pre1') + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + { + 'msg': 'connected', + 'session': msgs[0].get('session', None), + }, + ], + ) + + id_ = sockjs.call('login', 1, 2) # takes single argument + msgs = sockjs.recv() + self.assertEqual( + msgs, [ + { + 'msg': 'result', + 'error': { + 'error': 400, + 'reason': + 'login() takes exactly 2 arguments (3 given)', + }, + 'id': id_, + }, + ], + ) + + + def load_tests(loader, tests, pattern): """Specify which test cases to run.""" del pattern diff --git a/dddp/websocket.py b/dddp/websocket.py index b6ecfa0..940512f 100644 --- a/dddp/websocket.py +++ b/dddp/websocket.py @@ -14,19 +14,44 @@ from six.moves import range as irange import ejson import gevent import geventwebsocket +from django.conf import settings from django.core import signals from django.core.handlers.base import BaseHandler from django.core.handlers.wsgi import WSGIRequest from django.db import connection, transaction -from dddp import THREAD_LOCAL as this, alea, ADDED, CHANGED, REMOVED +from dddp import alea, this, ADDED, CHANGED, REMOVED, MeteorError -class MeteorError(Exception): +def safe_call(func, *args, **kwargs): + """ + Call `func(*args, **kwargs)` but NEVER raise an exception. - """MeteorError.""" + Useful in situations such as inside exception handlers where calls to + `logging.error` try to send email, but the SMTP server isn't always + availalbe and you don't want your exception handler blowing up. + """ + try: + return None, func(*args, **kwargs) + except Exception: # pylint: disable=broad-except + # something went wrong during the call, return a stack trace that can + # be dealt with by the caller + return traceback.format_exc(), None - pass + +def dprint(name, val): + """Debug print name and val.""" + from pprint import pformat + print( + '% 5s: %s' % ( + name, + '\n '.join( + pformat( + val, indent=4, width=75, + ).split('\n') + ), + ), + ) def validate_kwargs(func, kwargs): @@ -65,11 +90,12 @@ def validate_kwargs(func, kwargs): ] if missing: raise MeteorError( + 400, + func.err, 'Missing required arguments to %s: %s' % ( func_name, ' '.join(missing), ), - getattr(func, 'err', None), ) # figure out what is extra @@ -79,10 +105,9 @@ def validate_kwargs(func, kwargs): ] if extra: raise MeteorError( - 'Unknown arguments to %s: %s' % ( - func_name, - ' '.join(extra), - ), + 400, + func.err, + 'Unknown arguments to %s: %s' % (func_name, ' '.join(extra)), ) @@ -121,12 +146,11 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): this.ws = self this.send = self.send this.reply = self.reply - this.error = self.error self.logger = self.ws.logger self.remote_ids = collections.defaultdict(set) - # self._tx_buffer collects outgoing messages which must be sent in order + # `_tx_buffer` collects outgoing messages which must be sent in order self._tx_buffer = {} # track the head of the queue (buffer) and the next msg to be sent self._tx_buffer_id_gen = itertools.cycle(irange(sys.maxint)) @@ -139,7 +163,7 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): self.ws.environ, ) this.subs = {} - self.logger.info('+ %s OPEN', self) + safe_call(self.logger.info, '+ %s OPEN', self) self.send('o') self.send('a["{\\"server_id\\":\\"0\\"}"]') @@ -154,83 +178,144 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): self.connection.delete() self.connection = None signals.request_finished.send(sender=self.__class__) - self.logger.info('- %s %s', self, args or 'CLOSE') + safe_call(self.logger.info, '- %s %s', self, args or 'CLOSE') def on_message(self, message): """Process a message received from remote.""" if self.ws.closed: return None try: - self.logger.debug('< %s %r', self, message) - - # parse message set - try: - msgs = ejson.loads(message) - except ValueError as err: - self.error(400, 'Data is not valid EJSON') - return - if not isinstance(msgs, list): - self.error(400, 'Invalid EJSON messages') - return + safe_call(self.logger.debug, '< %s %r', self, message) # process individual messages - while msgs: - # parse message payload - raw = msgs.pop(0) - try: - data = ejson.loads(raw) - except (TypeError, ValueError) as err: - self.error(400, 'Data is not valid EJSON') - continue - if not isinstance(data, dict): - self.error(400, 'Invalid EJSON message payload', raw) - continue - try: - msg = data.pop('msg') - except KeyError: - self.error(400, 'Bad request', offendingMessage=data) - continue - # dispatch message - try: - self.dispatch(msg, data) - except MeteorError as err: - self.error(err) - except Exception as err: - traceback.print_exc() - self.error(err) + for data in self.ddp_frames_from_message(message): + self.process_ddp(data) # emit request_finished signal to close DB connections signals.request_finished.send(sender=self.__class__) - if msgs: - # yield to other greenlets before processing next msg - gevent.sleep() - except geventwebsocket.WebSocketError as err: + except geventwebsocket.WebSocketError: self.ws.close() + def ddp_frames_from_message(self, message): + """Yield DDP messages from a raw WebSocket message.""" + # parse message set + try: + msgs = ejson.loads(message) + except ValueError: + self.reply( + 'error', error=400, reason='Data is not valid EJSON', + ) + raise StopIteration + if not isinstance(msgs, list): + self.reply( + 'error', error=400, reason='Invalid EJSON messages', + ) + raise StopIteration + # process individual messages + while msgs: + # pop raw message from the list + raw = msgs.pop(0) + # parse message payload + try: + data = ejson.loads(raw) + except (TypeError, ValueError): + data = None + if not isinstance(data, dict): + self.reply( + 'error', error=400, + reason='Invalid SockJS DDP payload', + offendingMessage=raw, + ) + yield data + if msgs: + # yield to other greenlets before processing next msg + gevent.sleep() + + def process_ddp(self, data): + """Process a single DDP message.""" + msg_id = data.get('id', None) + try: + msg = data.pop('msg') + except KeyError: + self.reply( + 'error', reason='Bad request', + offendingMessage=data, + ) + return + try: + # dispatch message + self.dispatch(msg, data) + except Exception as err: # pylint: disable=broad-except + # This should be the only protocol exception handler + kwargs = { + 'msg': {'method': 'result'}.get(msg, 'error'), + } + if msg_id is not None: + kwargs['id'] = msg_id + if isinstance(err, MeteorError): + error = err.as_dict() + else: + error = { + 'error': 500, + 'reason': 'Internal server error', + } + if kwargs['msg'] == 'error': + kwargs.update(error) + else: + kwargs['error'] = error + if not isinstance(err, MeteorError): + # not a client error, should always be logged. + stack, _ = safe_call( + self.logger.error, '%r %r', msg, data, exc_info=1, + ) + if stack is not None: + # something went wrong while logging the error, revert to + # writing a stack trace to stderr. + traceback.print_exc(file=sys.stderr) + sys.stderr.write( + 'Additionally, while handling the above error the ' + 'following error was encountered:\n' + ) + sys.stderr.write(stack) + elif settings.DEBUG: + print('ERROR: %s' % err) + dprint('msg', msg) + dprint('data', data) + error.setdefault('details', traceback.format_exc()) + # print stack trace for client errors when DEBUG is True. + print(error['details']) + self.reply(**kwargs) + if msg_id and msg == 'method': + self.reply('updated', methods=[msg_id]) + @transaction.atomic def dispatch(self, msg, kwargs): """Dispatch msg to appropriate recv_foo handler.""" # enforce calling 'connect' first if self.connection is None and msg != 'connect': - self.error(400, 'Must connect first') + self.reply('error', reason='Must connect first') return + if msg == 'method': + if ( + 'method' not in kwargs + ) or ( + 'id' not in kwargs + ): + self.reply( + 'error', error=400, reason='Malformed method invocation', + ) + return # lookup method handler try: handler = getattr(self, 'recv_%s' % msg) except (AttributeError, UnicodeEncodeError): - print('Method not found: %s %r' % (msg, kwargs)) - self.error(404, 'Method not found', msg='result') - return + raise MeteorError(404, 'Method not found') # validate handler arguments validate_kwargs(handler, kwargs) # dispatch to handler - try: - handler(**kwargs) - except Exception as err: # print stack trace --> pylint: disable=W0703 - traceback.print_exc() - self.error(500, 'Internal server error', err) + handler(**kwargs) def send(self, data, tx_id=None): """Send `data` (raw string or EJSON payload) to WebSocket client.""" @@ -244,7 +329,7 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): # pull next message from buffer data = self._tx_buffer.pop(self._tx_next_id) if self._tx_buffer: - self.logger.debug('TX found %d', self._tx_next_id) + safe_call(self.logger.debug, 'TX found %d', self._tx_next_id) # advance next message ID self._tx_next_id = next(self._tx_next_id_gen) if not isinstance(data, basestring): @@ -270,15 +355,17 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): continue # client doesn't have this, don't send. data = 'a%s' % ejson.dumps([ejson.dumps(data)]) # send message - self.logger.debug('> %s %r', self, data) + safe_call(self.logger.debug, '> %s %r', self, data) try: self.ws.send(data) except geventwebsocket.WebSocketError: self.ws.close() + self._tx_buffer.clear() break num_waiting = len(self._tx_buffer) if num_waiting > 10: - self.logger.warn( + safe_call( + self.logger.warn, 'TX received %d, waiting for %d, have %d waiting: %r.', tx_id, self._tx_next_id, num_waiting, self._tx_buffer, ) @@ -288,52 +375,18 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): kwargs['msg'] = msg self.send(kwargs) - def error( - self, err, reason=None, detail=None, msg='error', exc_info=1, - **kwargs - ): - """Send EJSON error to remote.""" - if isinstance(err, MeteorError): - ( - err, reason, detail, kwargs, - ) = ( - err.args[:] + (None, None, None, None) - )[:4] - elif isinstance(err, Exception): - reason = str(err) - data = { - 'error': '%s' % (err or ''), - } - if reason: - if reason is Exception: - reason = str(reason) - data['reason'] = reason - if detail: - if isinstance(detail, Exception): - detail = str(detail) - data['detail'] = detail - if kwargs: - data.update(kwargs) - record = { - 'extra': { - 'request': this.request, - }, - } - self.logger.error('! %s %r', self, data, exc_info=exc_info, **record) - self.reply(msg, **data) - def recv_connect(self, version=None, support=None, session=None): """DDP connect handler.""" del session # Meteor doesn't even use this! if self.connection is not None: - self.error( + raise MeteorError( 400, 'Session already established.', - detail=self.connection.connection_id, + self.connection.connection_id, ) elif None in (version, support) or version not in self.versions: self.reply('failed', version=self.versions[0]) elif version not in support: - self.error(400, 'Client version/support mismatch.') + raise MeteorError(400, 'Client version/support mismatch.') else: from dddp.models import Connection cur = connection.cursor() @@ -352,6 +405,7 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): self.pgworker.connections[self.connection.pk] = self atexit.register(self.on_close, 'Shutting down.') self.reply('connected', session=self.connection.connection_id) + recv_connect.err = 'Malformed connect' def recv_ping(self, id_=None): """DDP ping handler.""" @@ -359,6 +413,7 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): self.reply('pong') else: self.reply('pong', id=id_) + recv_ping.err = 'Malformed ping' def recv_sub(self, id_, name, params): """DDP sub handler.""" @@ -371,6 +426,7 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): self.api.unsub(id_) else: self.reply('nosub') + recv_unsub.err = 'Malformed unsubscription' def recv_method(self, method, params, id_, randomSeed=None): """DDP method handler.""" diff --git a/docs/conf.py b/docs/conf.py index d60fa22..a0de59d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ settings.configure( 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': os.environ.get('PGDATABASE', 'django_ddp_docs_project'), - 'USER': os.environ.get('PGUSER', os.environ['LOGNAME']), + 'USER': os.environ.get('PGUSER', os.environ['USER']), 'PORT': int(os.environ.get('PGPORT', '0')) or None, 'PASSWORD': os.environ.get('PGPASSWORD', '') or None, 'HOST': os.environ.get('PGHOST', '') or None, @@ -90,7 +90,7 @@ copyright = u'2015, Tyson Clugg' # The short X.Y version. version = '0.19' # The full version, including alpha/beta/rc tags. -release = '0.19.0' +release = '0.19.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/requirements-test.txt b/requirements-test.txt index a5c10fa..19b5cbe 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,3 @@ # things required to run test suite requests==2.9.0 +websocket_client==0.34.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3c6e79c..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal=1 diff --git a/setup.py b/setup.py index 260f9bc..78a7f99 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,18 @@ #!/usr/bin/env python """Django/PostgreSQL implementation of the Meteor server.""" +# stdlib import os.path -import setuptools +import posixpath # all path specs in this file are UNIX-style paths +import shutil import subprocess from distutils import log from distutils.version import StrictVersion -from distutils.command.build import build +import setuptools.command.build_py +import setuptools.command.build_ext + +# pypi +import setuptools # setuptools 18.5 introduces support for the `platform_python_implementation` # environment marker: https://github.com/jaraco/setuptools/pull/28 @@ -15,28 +21,119 @@ __requires__ = 'setuptools>=18.5' assert StrictVersion(setuptools.__version__) >= StrictVersion('18.5'), \ 'Installation from source requires setuptools>=18.5.' +SETUP_DIR = os.path.dirname(__file__) -class Build(build): - """Build all files of a package.""" +class build_meteor(setuptools.command.build_py.build_py): + + """Build a Meteor project.""" + + user_options = [ + ('meteor=', None, 'path to `meteor` executable (default: meteor)'), + ('meteor-debug', None, 'meteor build with `--debug`'), + ('no-prune-npm', None, "don't prune meteor npm build directories"), + ('build-lib', 'd', 'directory to "build" (copy) to'), + ] + + negative_opt = [] + + meteor = None + meteor_debug = None + build_lib = None + package_dir = None + meteor_builds = None + no_prune_npm = None + inplace = None + + def initialize_options(self): + """Set command option defaults.""" + setuptools.command.build_py.build_py.initialize_options(self) + self.meteor = 'meteor' + self.meteor_debug = False + self.build_lib = None + self.package_dir = None + self.meteor_builds = [] + self.no_prune_npm = None + self.inplace = True + + def finalize_options(self): + """Update command options.""" + # Get all the information we need to install pure Python modules + # from the umbrella 'install' command -- build (source) directory, + # install (target) directory, and whether to compile .py files. + self.set_undefined_options( + 'build', + ('build_lib', 'build_lib'), + ) + self.set_undefined_options( + 'build_py', + ('package_dir', 'package_dir'), + ) + setuptools.command.build_py.build_py.finalize_options(self) + + @staticmethod + def has_meteor_builds(distribution): + """Returns `True` if distribution has meteor projects to be built.""" + return bool( + distribution.command_options['build_meteor']['meteor_builds'] + ) + + def get_package_dir(self, package): + res = setuptools.command.build_py.orig.build_py.get_package_dir( + self, package, + ) + if self.distribution.src_root is not None: + return os.path.join(self.distribution.src_root, res) + return res def run(self): - """Build our package.""" - cmdline = [ - 'meteor', - 'build', - '--directory', - '../build', - ] - meteor_dir = os.path.join( - os.path.dirname(__file__), - 'dddp', - 'test', - 'meteor_todos', + """Peform build.""" + for (package, source, target, extra_args) in self.meteor_builds: + src_dir = self.get_package_dir(package) + # convert UNIX-style paths to directory names + project_dir = self.path_to_dir(src_dir, source) + target_dir = self.path_to_dir(src_dir, target) + output_dir = self.path_to_dir( + os.path.abspath(SETUP_DIR if self.inplace else self.build_lib), + target_dir, + ) + # construct command line. + cmdline = [self.meteor, 'build', '--directory', output_dir] + no_prune_npm = self.no_prune_npm + if extra_args[:1] == ['--no-prune-npm']: + no_prune_npm = True + extra_args[:1] = [] + if self.meteor_debug and '--debug' not in cmdline: + cmdline.append('--debug') + cmdline.extend(extra_args) + # execute command + log.info( + 'building meteor app %r (%s)', project_dir, ' '.join(cmdline), + ) + subprocess.check_call(cmdline, cwd=project_dir) + if not no_prune_npm: + # django-ddp doesn't use bundle/programs/server/npm cruft + npm_build_dir = os.path.join( + output_dir, 'bundle', 'programs', 'server', 'npm', + ) + log.info('pruning meteor npm build %r', npm_build_dir) + shutil.rmtree(npm_build_dir) + + @staticmethod + def path_to_dir(*path_args): + """Convert a UNIX-style path into platform specific directory spec.""" + return os.path.join( + *list(path_args[:-1]) + path_args[-1].split(posixpath.sep) ) - log.info('Building meteor app %r (%s)', meteor_dir, ' '.join(cmdline)) - subprocess.check_call(cmdline, cwd=meteor_dir) - return build.run(self) + + +class build_ext(setuptools.command.build_ext.build_ext): + + def run(self): + if build_meteor.has_meteor_builds(self.distribution): + self.reinitialize_command('build_meteor', inplace=True) + self.run_command('build_meteor') + return setuptools.command.build_ext.build_ext.run(self) CLASSIFIERS = [ @@ -81,7 +178,7 @@ CLASSIFIERS = [ setuptools.setup( name='django-ddp', - version='0.19.0', + version='0.19.1', description=__doc__, long_description=open('README.rst').read(), author='Tyson Clugg', @@ -92,16 +189,15 @@ setuptools.setup( 'liveupdate live-update livequery live-query' ], license='MIT', - packages=setuptools.find_packages(), + packages=setuptools.find_packages(exclude=['tests*']), include_package_data=True, # install data files specified in MANIFEST.in - zip_safe=False, # TODO: Move dddp.test into it's own package. + zip_safe=True, setup_requires=[ # packages required to run the setup script __requires__, ], install_requires=[ 'Django>=1.8', - 'django-dbarray>=0.2', 'meteor-ejson>=1.0', 'psycogreen>=1.0', 'pybars3>=0.9.1', @@ -149,11 +245,23 @@ setuptools.setup( ], }, classifiers=CLASSIFIERS, - test_suite='dddp.test.run_tests', + test_suite='tests.manage.run_tests', tests_require=[ 'requests', + 'websocket_client', ], cmdclass={ - 'build': Build, + 'build_ext': build_ext, + 'build_meteor': build_meteor, + }, + options={ + 'bdist_wheel': { + 'universal': '1', + }, + 'build_meteor': { + 'meteor_builds': [ + ('tests', 'meteor_todos', 'build', []), + ], + }, }, ) diff --git a/dddp/test/django_todos/__init__.py b/tests/__init__.py similarity index 100% rename from dddp/test/django_todos/__init__.py rename to tests/__init__.py diff --git a/dddp/test/django_todos/migrations/__init__.py b/tests/django_todos/__init__.py similarity index 100% rename from dddp/test/django_todos/migrations/__init__.py rename to tests/django_todos/__init__.py diff --git a/dddp/test/django_todos/admin.py b/tests/django_todos/admin.py similarity index 100% rename from dddp/test/django_todos/admin.py rename to tests/django_todos/admin.py diff --git a/dddp/test/django_todos/ddp.py b/tests/django_todos/ddp.py similarity index 71% rename from dddp/test/django_todos/ddp.py rename to tests/django_todos/ddp.py index b4bc31b..b4d04d8 100644 --- a/dddp/test/django_todos/ddp.py +++ b/tests/django_todos/ddp.py @@ -1,5 +1,7 @@ +from __future__ import absolute_import, unicode_literals + from dddp.api import API, Collection, Publication -from dddp.test.django_todos import models +from django_todos import models class Task(Collection): diff --git a/dddp/test/django_todos/migrations/0001_initial.py b/tests/django_todos/migrations/0001_initial.py similarity index 100% rename from dddp/test/django_todos/migrations/0001_initial.py rename to tests/django_todos/migrations/0001_initial.py diff --git a/dddp/test/test_project/__init__.py b/tests/django_todos/migrations/__init__.py similarity index 100% rename from dddp/test/test_project/__init__.py rename to tests/django_todos/migrations/__init__.py diff --git a/dddp/test/django_todos/models.py b/tests/django_todos/models.py similarity index 100% rename from dddp/test/django_todos/models.py rename to tests/django_todos/models.py diff --git a/dddp/test/django_todos/tests.py b/tests/django_todos/tests.py similarity index 91% rename from dddp/test/django_todos/tests.py rename to tests/django_todos/tests.py index 70500dc..a4b6997 100644 --- a/dddp/test/django_todos/tests.py +++ b/tests/django_todos/tests.py @@ -4,7 +4,7 @@ import doctest import os import unittest -os.environ['DJANGO_SETTINGS_MODULE'] = 'dddp.test.test_project.settings' +os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_project.settings' DOCTEST_MODULES = [ ] diff --git a/tests/django_todos/views.py b/tests/django_todos/views.py new file mode 100644 index 0000000..dea83e1 --- /dev/null +++ b/tests/django_todos/views.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import +import os.path + +from dddp.views import MeteorView +import tests + + +class MeteorTodos(MeteorView): + """Meteor Todos.""" + + json_path = os.path.join( + os.path.dirname(tests.__file__), + 'build', 'bundle', 'star.json' + ) diff --git a/tests/manage.py b/tests/manage.py new file mode 100755 index 0000000..b6f1a70 --- /dev/null +++ b/tests/manage.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +"""Entry point for Django DDP test project.""" +import os +import sys + +import dddp +dddp.greenify() + +os.environ['DJANGO_SETTINGS_MODULE'] = 'test_project.settings' +sys.path.insert(0, os.path.dirname(__file__)) + + +def run_tests(): + """Run the test suite.""" + import django + from django.test.runner import DiscoverRunner + django.setup() + test_runner = DiscoverRunner(verbosity=2, interactive=False) + failures = test_runner.run_tests(['.']) + sys.exit(bool(failures)) + + +def main(args): # pragma: no cover + """Execute a management command.""" + from django.core.management import execute_from_command_line + execute_from_command_line(args) + + +if __name__ == "__main__": # pragma: no cover + main(sys.argv) diff --git a/dddp/test/meteor_todos/.meteor/.finished-upgraders b/tests/meteor_todos/.meteor/.finished-upgraders similarity index 100% rename from dddp/test/meteor_todos/.meteor/.finished-upgraders rename to tests/meteor_todos/.meteor/.finished-upgraders diff --git a/dddp/test/meteor_todos/.meteor/.gitignore b/tests/meteor_todos/.meteor/.gitignore similarity index 100% rename from dddp/test/meteor_todos/.meteor/.gitignore rename to tests/meteor_todos/.meteor/.gitignore diff --git a/dddp/test/meteor_todos/.meteor/.id b/tests/meteor_todos/.meteor/.id similarity index 100% rename from dddp/test/meteor_todos/.meteor/.id rename to tests/meteor_todos/.meteor/.id diff --git a/dddp/test/meteor_todos/.meteor/packages b/tests/meteor_todos/.meteor/packages similarity index 100% rename from dddp/test/meteor_todos/.meteor/packages rename to tests/meteor_todos/.meteor/packages diff --git a/dddp/test/meteor_todos/.meteor/platforms b/tests/meteor_todos/.meteor/platforms similarity index 100% rename from dddp/test/meteor_todos/.meteor/platforms rename to tests/meteor_todos/.meteor/platforms diff --git a/dddp/test/meteor_todos/.meteor/release b/tests/meteor_todos/.meteor/release similarity index 100% rename from dddp/test/meteor_todos/.meteor/release rename to tests/meteor_todos/.meteor/release diff --git a/dddp/test/meteor_todos/.meteor/versions b/tests/meteor_todos/.meteor/versions similarity index 100% rename from dddp/test/meteor_todos/.meteor/versions rename to tests/meteor_todos/.meteor/versions diff --git a/dddp/test/meteor_todos/meteor_todos.css b/tests/meteor_todos/meteor_todos.css similarity index 100% rename from dddp/test/meteor_todos/meteor_todos.css rename to tests/meteor_todos/meteor_todos.css diff --git a/dddp/test/meteor_todos/meteor_todos.html b/tests/meteor_todos/meteor_todos.html similarity index 100% rename from dddp/test/meteor_todos/meteor_todos.html rename to tests/meteor_todos/meteor_todos.html diff --git a/dddp/test/meteor_todos/meteor_todos.js b/tests/meteor_todos/meteor_todos.js similarity index 100% rename from dddp/test/meteor_todos/meteor_todos.js rename to tests/meteor_todos/meteor_todos.js diff --git a/tests/test_project/__init__.py b/tests/test_project/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dddp/test/test_project/settings.py b/tests/test_project/settings.py similarity index 93% rename from dddp/test/test_project/settings.py rename to tests/test_project/settings.py index 781c342..9e1cb92 100644 --- a/dddp/test/test_project/settings.py +++ b/tests/test_project/settings.py @@ -20,7 +20,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(__file__)) SECRET_KEY = 'z@akz#7+cp9w!7%=%kqec79ltlzdn5p&__=(th8^&*t)vo4p35' # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False TEMPLATE_DEBUG = True @@ -38,7 +38,7 @@ INSTALLED_APPS = ( 'django.contrib.staticfiles', 'dddp', 'dddp.accounts', - 'dddp.test.django_todos', + 'django_todos', ) MIDDLEWARE_CLASSES = ( @@ -51,9 +51,9 @@ MIDDLEWARE_CLASSES = ( 'django.middleware.clickjacking.XFrameOptionsMiddleware', ) -ROOT_URLCONF = 'dddp.test.test_project.urls' +ROOT_URLCONF = 'test_project.urls' -WSGI_APPLICATION = 'dddp.test.test_project.wsgi.application' +WSGI_APPLICATION = 'test_project.wsgi.application' # Database diff --git a/dddp/test/test_project/urls.py b/tests/test_project/urls.py similarity index 60% rename from dddp/test/test_project/urls.py rename to tests/test_project/urls.py index 6547d18..6f2b3d6 100644 --- a/dddp/test/test_project/urls.py +++ b/tests/test_project/urls.py @@ -1,22 +1,15 @@ """Django DDP test project - URL configuraiton.""" -import os.path from django.conf import settings from django.conf.urls import patterns, include, url from django.contrib import admin -from dddp.views import MeteorView -import dddp.test +from django_todos.views import MeteorTodos -app = MeteorView.as_view( - json_path=os.path.join( - os.path.dirname(dddp.test.__file__), - 'build', 'bundle', 'star.json' - ), -) -urlpatterns = patterns('', +urlpatterns = patterns( + '', # Examples: - # url(r'^$', 'dddp.test.test_project.views.home', name='home'), + # url(r'^$', 'test_project.views.home', name='home'), # url(r'^blog/', include('blog.urls')), url(r'^admin/', include(admin.site.urls)), @@ -29,5 +22,5 @@ urlpatterns = patterns('', }, ), # all remaining URLs routed to Meteor app. - url(r'^(?P.*)$', app), + url(r'^(?P.*)$', MeteorTodos.as_view()), ) diff --git a/dddp/test/test_project/wsgi.py b/tests/test_project/wsgi.py similarity index 70% rename from dddp/test/test_project/wsgi.py rename to tests/test_project/wsgi.py index d076675..a7a5657 100644 --- a/dddp/test/test_project/wsgi.py +++ b/tests/test_project/wsgi.py @@ -1,5 +1,5 @@ """ -WSGI config for dddp.test.test_project project. +WSGI config for test_project project. It exposes the WSGI callable as a module-level variable named ``application``. @@ -8,7 +8,7 @@ https://docs.djangoproject.com/en/1.7/howto/deployment/wsgi/ """ import os -os.environ["DJANGO_SETTINGS_MODULE"] = "dddp.test.test_project.settings" +os.environ["DJANGO_SETTINGS_MODULE"] = "test_project.settings" from django.core.wsgi import get_wsgi_application application = get_wsgi_application() diff --git a/tox.ini b/tox.ini index ae3a0ab..e81c998 100644 --- a/tox.ini +++ b/tox.ini @@ -8,9 +8,6 @@ # require tox 2.1.1 or later minversion=2.1.1 -# don't fail if missing a python version specified in envlist -skip_missing_interpreters=True - # list of environments to run by default envlist = lint @@ -123,7 +120,7 @@ install_command=sh -c 'pip install -U "setuptools>=18.5" "wheel>=0.25.0" "pip>=7 whitelist_externals=sh commands = - check-manifest --ignore "dddp/test/build*,dddp/test/meteor_todos/.meteor/local*" + check-manifest {envpython} setup.py --no-user-cfg sdist --dist-dir={toxinidir}/dist/ {envpython} setup.py --no-user-cfg bdist_wheel --dist-dir={toxinidir}/dist/ sh -c "cd docs && sphinx-build -b html -d _build/doctrees -D latex_paper_size=a4 . _build/html"