diff --git a/CHANGES.rst b/CHANGES.rst index 88647f4..2eae416 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,28 @@ Change Log ========== +0.12.0 +------ +* Get path to `star.json` from view config (defined in your urls.py) + instead of from settings. +* Dropped `dddp.server.views`, use `dddp.views` instead. + +0.11.0 +------ +* Support more than 8KB of change data by splitting large payloads into + multiple chunks. + +0.10.2 +------ +* Add `Logs` publication that can be configured to emit logs via DDP + through the use of the `dddp.logging.DDPHandler` log handler. +* Add option to dddp daemon to provide a BackdoorServer (telnet) for + interactive debugging (REPL) at runtime. + +0.10.1 +------ +* Bugfix dddp.accounts forgot_password feature. + 0.10.0 ------ * Stop processing request middleware upon connection - see diff --git a/README.rst b/README.rst index afd6ece..367cc6e 100644 --- a/README.rst +++ b/README.rst @@ -103,6 +103,19 @@ Add ddp.py to your Django application: [Book, Author, AllBooks, BooksByAuthorEmail] ) +Start the Django DDP service: + +.. code:: sh + + DJANGO_SETTINGS_MODULE=myproject.settings dddp + + +Using django-ddp as a secondary DDP connection (RAPID DEVELOPMENT) +------------------------------------------------------------------ + +Running in this manner allows rapid development through use of the hot +code push features provided by Meteor. + Connect your Meteor application to the Django DDP service: .. code:: javascript @@ -118,19 +131,53 @@ Connect your Meteor application to the Django DDP service: Django.subscribe('BooksByAuthorEmail', 'janet@evanovich.com'); } -Start the Django DDP service: - -.. code:: sh - - DJANGO_SETTINGS_MODULE=myproject.settings dddp - -In a separate terminal, start Meteor (from within your meteor -application directory): +Start Meteor (from within your meteor application directory): .. code:: sh meteor +Using django-ddp as the primary DDP connection (RECOMMENDED) +------------------------------------------------------------ + +If you'd prefer to not have two DDP connections (one to Meteor and one +to django-ddp) you can set the `DDP_DEFAULT_CONNECTION_URL` environment +variable to use the specified URL as the primary DDP connection in +Meteor. When doing this, you won't need to use `DDP.connect(...)` or +specify `{connection: Django}` on your collections. Running with +django-ddp as the primary connection is recommended, and indeed required +if you wish to use `dddp.accounts` to provide authentication using +`django.contrib.auth` to your meteor app. + +.. code:: sh + + DDP_DEFAULT_CONNECTION_URL=http://localhost:8000/ meteor + + +Serving your Meteor applications from django-ddp +------------------------------------------------ + +First, you will need to build your meteor app into a directory (examples +below assume target directory named `myapp`): + +.. code:: sh + + meteor build ../myapp + +Then, add a MeteorView to your urls.py: + +.. code:: python + + from dddp.views import MeteorView + + urlpatterns = patterns( + url('^(?P/.*)$', MeteorView.as_view( + json_path=os.path.join( + settings.PROJ_ROOT, 'myapp', 'bundle', 'star.json', + ), + ), + ) + Adding API endpoints (server method definitions) ------------------------------------------------ diff --git a/dddp/accounts/ddp.py b/dddp/accounts/ddp.py index 05d328e..d49b0e8 100644 --- a/dddp/accounts/ddp.py +++ b/dddp/accounts/ddp.py @@ -538,7 +538,7 @@ class Auth(APIMixin): except self.user_model.DoesNotExist: self.auth_failed() - days_valid = HASH_DAYS_VALID[HashPurpose.PASSWORD_RESET], + days_valid = HASH_DAYS_VALID[HashPurpose.PASSWORD_RESET] token = get_user_token( user=user, purpose=HashPurpose.PASSWORD_RESET, days_valid=days_valid, diff --git a/dddp/api.py b/dddp/api.py index 131abb0..9153c43 100644 --- a/dddp/api.py +++ b/dddp/api.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, unicode_literals, print_function import collections from copy import deepcopy import traceback +import uuid # requirements import dbarray @@ -599,6 +600,7 @@ class DDP(APIMixin): 'params_ejson': ejson.dumps(params), }, ) + this.subs.setdefault(sub.publication, set()).add(sub.pk) if not created: if not silent: this.send({'msg': 'ready', 'subs': [id_]}) @@ -644,6 +646,7 @@ class DDP(APIMixin): for obj in qs: payload = col.obj_change_as_msg(obj, REMOVED, meteor_ids) this.send(payload) + this.subs[sub.publication].remove(sub.pk) sub.delete() if not silent: this.send({'msg': 'nosub', 'id': id_}) @@ -894,13 +897,32 @@ class DDP(APIMixin): if my_connection_id in connection_ids: # msg must go to connection that initiated the change payload['_tx_id'] = this.ws.get_tx_id() + # header is sent in every payload + header = { + 'uuid': uuid.uuid1().int, # UUID1 should be unique + 'seq': 1, # increments for each 8KB chunk + 'fin': 0, # zero if more chunks expected, 1 if last chunk. + } + data = ejson.dumps(payload) cursor = connections[using].cursor() - cursor.execute( - 'NOTIFY "ddp", %s', - [ - ejson.dumps(payload), - ], - ) + while data: + hdr = ejson.dumps(header) + # use all available payload space for chunk + max_len = 8000 - len(hdr) - 100 + # take a chunk from data + chunk, data = data[:max_len], data[max_len:] + if not data: + # last chunk, set fin=1. + header['fin'] = 1 + hdr = ejson.dumps(header) + # print('NOTIFY: %s' % hdr) + cursor.execute( + 'NOTIFY "ddp", %s', + [ + '%s|%s' % (hdr, chunk), # pipe separates hdr|chunk. + ], + ) + header['seq'] += 1 # increment sequence. API = DDP() diff --git a/dddp/ddp.py b/dddp/ddp.py new file mode 100644 index 0000000..0af0f11 --- /dev/null +++ b/dddp/ddp.py @@ -0,0 +1,22 @@ +from dddp import THREAD_LOCAL as this +from dddp.api import API, Publication +from django.contrib import auth + + +class Logs(Publication): + + users = auth.get_user_model() + + def get_queries(self): + user_pk = getattr(this, 'user_id', False) + if user_pk: + if self.users.objects.filter( + pk=user_pk, + is_active=True, + is_superuser=True, + ).exists(): + return [] + raise ValueError('User not permitted.') + + +API.register([Logs]) diff --git a/dddp/logging.py b/dddp/logging.py new file mode 100644 index 0000000..147873f --- /dev/null +++ b/dddp/logging.py @@ -0,0 +1,39 @@ +"""Django DDP logging helpers.""" +from __future__ import absolute_import, print_function + +import logging + +from dddp import THREAD_LOCAL as this, meteor_random_id, ADDED + + + + +class DDPHandler(logging.Handler): + + """Logging handler that streams log events via DDP to the current client.""" + + def __init__(self, *args, **kwargs): + print(self.__class__, args, kwargs) + self.logger = logging.getLogger('django.db.backends') + self.logger.info('Test') + super(DDPHandler, self).__init__(*args, **kwargs) + + def emit(self, record): + """Emit a formatted log record via DDP.""" + if getattr(this, 'subs', {}).get('Logs', False): + this.send({ + 'msg': ADDED, + 'collection': 'logs', + 'id': meteor_random_id('/collection/logs'), + 'fields': { + # 'name': record.name, + # 'levelno': record.levelno, + 'levelname': record.levelname, + # 'pathname': record.pathname, + # 'lineno': record.lineno, + 'msg': record.msg, + 'args': record.args, + # 'exc_info': record.exc_info, + # 'funcName': record.funcName, + }, + }) diff --git a/dddp/main.py b/dddp/main.py index 2b0f863..e8ab34a 100644 --- a/dddp/main.py +++ b/dddp/main.py @@ -60,7 +60,7 @@ def ddpp_sockjs_info(environ, start_response): ])) -def serve(listen, debug=False, **ssl_args): +def serve(listen, debug=False, verbosity=1, debug_port=0, **ssl_args): """Spawn greenlets for handling websockets and PostgreSQL calls.""" import signal from django.apps import apps @@ -99,7 +99,7 @@ def serve(listen, debug=False, **ssl_args): ) # setup WebSocketServer to dispatch web requests - webservers = [ + servers = [ geventwebsocket.WebSocketServer( (host, port), resource, @@ -113,8 +113,8 @@ def serve(listen, debug=False, **ssl_args): def killall(*args, **kwargs): """Kill all green threads.""" pgworker.stop() - for webserver in webservers: - webserver.stop() + for server in servers: + server.stop() # die gracefully with SIGINT or SIGQUIT gevent.signal(signal.SIGINT, killall) @@ -133,19 +133,38 @@ def serve(listen, debug=False, **ssl_args): ) # start greenlets + if debug_port: + from gevent.backdoor import BackdoorServer + servers.append( + BackdoorServer( + ('127.0.0.1', debug_port), + banner='Django DDP', + locals={ + 'servers': servers, + 'pgworker': pgworker, + 'killall': killall, + 'api': api, + 'resource': resource, + 'settings': settings, + 'wsgi_app': wsgi_app, + 'wsgi_name': wsgi_name, + }, + ) + ) + pgworker.start() print('=> Started PostgresGreenlet.') - web_threads = [ - gevent.spawn(webserver.serve_forever) - for webserver - in webservers + threads = [ + gevent.spawn(server.serve_forever) + for server + in servers ] print('=> Started DDPWebSocketApplication.') print('=> Started your app (%s).' % wsgi_name) print('') for host, port in listen: print('=> App running at: http://%s:%d/' % (host, port)) - gevent.joinall(web_threads) + gevent.joinall(threads) pgworker.stop() gevent.joinall([pgworker]) @@ -190,6 +209,14 @@ def main(): import argparse parser = argparse.ArgumentParser(description=__doc__) django = parser.add_argument_group('Django Options') + django.add_argument( + '--verbosity', '-v', metavar='VERBOSITY', dest='verbosity', type=int, + default=1, + ) + django.add_argument( + '--debug-port', metavar='DEBUG_PORT', dest='debug_port', type=int, + default=0, + ) django.add_argument( '--settings', metavar='SETTINGS', dest='settings', help="The Python path to a settings module, e.g. " @@ -218,8 +245,10 @@ def main(): os.environ['DJANGO_SETTINGS_MODULE'] = namespace.settings serve( namespace.listen or [Addr('localhost', 8000)], + debug_port=namespace.debug_port, keyfile=namespace.keyfile, certfile=namespace.certfile, + verbosity=namespace.verbosity, ) diff --git a/dddp/postgres.py b/dddp/postgres.py index 2c2404c..51844d0 100644 --- a/dddp/postgres.py +++ b/dddp/postgres.py @@ -22,6 +22,7 @@ class PostgresGreenlet(gevent.Greenlet): # queues for processing incoming sub/unsub requests and processing self.connections = {} + self.chunks = {} self._stop_event = gevent.event.Event() # connect to DB in async mode @@ -62,7 +63,29 @@ class PostgresGreenlet(gevent.Greenlet): "Got NOTIFY (pid=%d, payload=%r)", notify.pid, notify.payload, ) - data = ejson.loads(notify.payload) + + # read the header and check seq/fin. + hdr, chunk = notify.payload.split('|', 1) + # print('RECEIVE: %s' % hdr) + header = ejson.loads(hdr) + uuid = header['uuid'] + size, chunks = self.chunks.setdefault(uuid, [0, {}]) + if header['fin']: + size = self.chunks[uuid][0] = header['seq'] + + # stash the chunk + chunks[header['seq']] = chunk + + if len(chunks) != size: + # haven't got all the chunks yet + continue # process next NOTIFY in loop + + # got the last chunk -> process it. + data = ''.join( + chunk for _, chunk in sorted(chunks.items()) + ) + del self.chunks[uuid] # don't forget to cleanup! + data = ejson.loads(data) sender = data.pop('_sender', None) tx_id = data.pop('_tx_id', None) for connection_id in data.pop('_connection_ids'): diff --git a/dddp/server/__init__.py b/dddp/server/__init__.py deleted file mode 100644 index cea26b4..0000000 --- a/dddp/server/__init__.py +++ /dev/null @@ -1 +0,0 @@ -default_app_config = 'dddp.server.apps.ServerConfig' diff --git a/dddp/server/apps.py b/dddp/server/apps.py deleted file mode 100644 index 4a742e9..0000000 --- a/dddp/server/apps.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Django DDP Server app config.""" -from __future__ import print_function, absolute_import, unicode_literals - -import mimetypes - -from django.apps import AppConfig - - -class ServerConfig(AppConfig): - - """Django config for dddp.server app.""" - - name = 'dddp.server' - verbose_name = 'Django DDP Meteor Web Server' - - def ready(self): - """Configure Django DDP server app.""" - mimetypes.init() # read and process /etc/mime.types diff --git a/dddp/server/views.py b/dddp/views.py similarity index 98% rename from dddp/server/views.py rename to dddp/views.py index 318cfb3..a631b95 100644 --- a/dddp/server/views.py +++ b/dddp/views.py @@ -1,9 +1,7 @@ """Django DDP Server views.""" from __future__ import print_function, absolute_import, unicode_literals -import io import mimetypes -import os.path from ejson import dumps, loads from django.conf import settings @@ -63,6 +61,9 @@ class MeteorView(View): # super(...).__init__ assigns kwargs to instance. super(MeteorView, self).__init__(**kwargs) + # read and process /etc/mime.types + mimetypes.init() + self.url_map = {} # process `star_json` @@ -187,8 +188,6 @@ class MeteorView(View): def get(self, request, path): """Return HTML (or other related content) for Meteor.""" - if path[:1] != '/': - path = '/%s' % path if path == '/meteor_runtime_config.js': config = { 'DDP_DEFAULT_CONNECTION_URL': request.build_absolute_uri('/'), diff --git a/dddp/websocket.py b/dddp/websocket.py index b599e29..0676363 100644 --- a/dddp/websocket.py +++ b/dddp/websocket.py @@ -106,7 +106,6 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): version = None support = None connection = None - subs = None remote_ids = None base_handler = BaseHandler() @@ -131,7 +130,7 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication): '{0[REMOTE_ADDR]}:{0[REMOTE_PORT]}'.format( self.ws.environ, ) - self.subs = {} + this.subs = {} self.logger.info('+ %s OPEN', self) self.send('o') self.send('a["{\\"server_id\\":\\"0\\"}"]') diff --git a/setup.py b/setup.py index a736cae..c84f740 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup, find_packages setup( name='django-ddp', - version='0.10.0', + version='0.12.0', description=__doc__, long_description=open('README.rst').read(), author='Tyson Clugg',