Merge branch 'develop' into feature/AleaIdField_as_primary_key

Conflicts:
	dddp/server/views.py
	dddp/views.py

CHANGES.rst

@@ -1,6 +1,28 @@
 Change Log
 ==========
 
+0.12.0
+------
+* Get path to `star.json` from view config (defined in your urls.py) 
+  instead of from settings.
+* Dropped `dddp.server.views`, use `dddp.views` instead.
+
+0.11.0
+------
+* Support more than 8KB of change data by splitting large payloads into 
+  multiple chunks.
+
+0.10.2
+------
+* Add `Logs` publication that can be configured to emit logs via DDP 
+  through the use of the `dddp.logging.DDPHandler` log handler.
+* Add option to dddp daemon to provide a BackdoorServer (telnet) for 
+  interactive debugging (REPL) at runtime.
+
+0.10.1
+------
+* Bugfix dddp.accounts forgot_password feature.
+
 0.10.0
 ------
 * Stop processing request middleware upon connection - see

README.rst

@@ -103,6 +103,19 @@ Add ddp.py to your Django application:
         [Book, Author, AllBooks, BooksByAuthorEmail]
     )
 
+Start the Django DDP service:
+
+.. code:: sh
+
+    DJANGO_SETTINGS_MODULE=myproject.settings dddp
+
+
+Using django-ddp as a secondary DDP connection (RAPID DEVELOPMENT)
+------------------------------------------------------------------
+
+Running in this manner allows rapid development through use of the hot 
+code push features provided by Meteor.
+
 Connect your Meteor application to the Django DDP service:
 
 .. code:: javascript
@@ -118,19 +131,53 @@ Connect your Meteor application to the Django DDP service:
         Django.subscribe('BooksByAuthorEmail', 'janet@evanovich.com');
     }
 
-Start the Django DDP service:
-
-.. code:: sh
-
-    DJANGO_SETTINGS_MODULE=myproject.settings dddp
-
-In a separate terminal, start Meteor (from within your meteor
-application directory):
+Start Meteor (from within your meteor application directory):
 
 .. code:: sh
 
     meteor
 
+Using django-ddp as the primary DDP connection (RECOMMENDED)
+------------------------------------------------------------
+
+If you'd prefer to not have two DDP connections (one to Meteor and one 
+to django-ddp) you can set the `DDP_DEFAULT_CONNECTION_URL` environment 
+variable to use the specified URL as the primary DDP connection in 
+Meteor.  When doing this, you won't need to use `DDP.connect(...)` or 
+specify `{connection: Django}` on your collections.  Running with 
+django-ddp as the primary connection is recommended, and indeed required 
+if you wish to use `dddp.accounts` to provide authentication using 
+`django.contrib.auth` to your meteor app.
+
+.. code:: sh
+
+    DDP_DEFAULT_CONNECTION_URL=http://localhost:8000/ meteor
+
+
+Serving your Meteor applications from django-ddp
+------------------------------------------------
+
+First, you will need to build your meteor app into a directory (examples 
+below assume target directory named `myapp`):
+
+.. code:: sh
+
+    meteor build ../myapp
+
+Then, add a MeteorView to your urls.py:
+
+.. code:: python
+
+    from dddp.views import MeteorView
+
+    urlpatterns = patterns(
+        url('^(?P<path>/.*)$', MeteorView.as_view(
+            json_path=os.path.join(
+                settings.PROJ_ROOT, 'myapp', 'bundle', 'star.json',
+            ),
+        ),
+    )
+
 
 Adding API endpoints (server method definitions)
 ------------------------------------------------

dddp/accounts/ddp.py

@@ -538,7 +538,7 @@ class Auth(APIMixin):
         except self.user_model.DoesNotExist:
             self.auth_failed()
 
-        days_valid = HASH_DAYS_VALID[HashPurpose.PASSWORD_RESET],
+        days_valid = HASH_DAYS_VALID[HashPurpose.PASSWORD_RESET]
         token = get_user_token(
             user=user, purpose=HashPurpose.PASSWORD_RESET,
             days_valid=days_valid,

dddp/api.py

@@ -5,6 +5,7 @@ from __future__ import absolute_import, unicode_literals, print_function
 import collections
 from copy import deepcopy
 import traceback
+import uuid
 
 # requirements
 import dbarray
@@ -599,6 +600,7 @@ class DDP(APIMixin):
                 'params_ejson': ejson.dumps(params),
             },
         )
+        this.subs.setdefault(sub.publication, set()).add(sub.pk)
         if not created:
             if not silent:
                 this.send({'msg': 'ready', 'subs': [id_]})
@@ -644,6 +646,7 @@ class DDP(APIMixin):
             for obj in qs:
                 payload = col.obj_change_as_msg(obj, REMOVED, meteor_ids)
                 this.send(payload)
+        this.subs[sub.publication].remove(sub.pk)
         sub.delete()
         if not silent:
             this.send({'msg': 'nosub', 'id': id_})
@@ -894,13 +897,32 @@ class DDP(APIMixin):
                     if my_connection_id in connection_ids:
                         # msg must go to connection that initiated the change
                         payload['_tx_id'] = this.ws.get_tx_id()
+                # header is sent in every payload
+                header = {
+                    'uuid': uuid.uuid1().int,  # UUID1 should be unique
+                    'seq': 1,  # increments for each 8KB chunk
+                    'fin': 0,  # zero if more chunks expected, 1 if last chunk.
+                }
+                data = ejson.dumps(payload)
                 cursor = connections[using].cursor()
-                cursor.execute(
-                    'NOTIFY "ddp", %s',
-                    [
-                        ejson.dumps(payload),
-                    ],
-                )
+                while data:
+                    hdr = ejson.dumps(header)
+                    # use all available payload space for chunk
+                    max_len = 8000 - len(hdr) - 100
+                    # take a chunk from data
+                    chunk, data = data[:max_len], data[max_len:]
+                    if not data:
+                        # last chunk, set fin=1.
+                        header['fin'] = 1
+                        hdr = ejson.dumps(header)
+                    # print('NOTIFY: %s' % hdr)
+                    cursor.execute(
+                        'NOTIFY "ddp", %s',
+                        [
+                            '%s|%s' % (hdr, chunk),  # pipe separates hdr|chunk.
+                        ],
+                    )
+                    header['seq'] += 1  # increment sequence.
 
 
 API = DDP()

dddp/ddp.py

@@ -0,0 +1,22 @@
+from dddp import THREAD_LOCAL as this
+from dddp.api import API, Publication
+from django.contrib import auth
+
+
+class Logs(Publication):
+
+    users = auth.get_user_model()
+
+    def get_queries(self):
+        user_pk = getattr(this, 'user_id', False)
+        if user_pk:
+            if self.users.objects.filter(
+                pk=user_pk,
+                is_active=True,
+                is_superuser=True,
+            ).exists():
+                return []
+        raise ValueError('User not permitted.')
+
+
+API.register([Logs])

dddp/logging.py

@@ -0,0 +1,39 @@
+"""Django DDP logging helpers."""
+from __future__ import absolute_import, print_function
+
+import logging
+
+from dddp import THREAD_LOCAL as this, meteor_random_id, ADDED
+
+
+
+
+class DDPHandler(logging.Handler):
+
+    """Logging handler that streams log events via DDP to the current client."""
+
+    def __init__(self, *args, **kwargs):
+        print(self.__class__, args, kwargs)
+        self.logger = logging.getLogger('django.db.backends')
+        self.logger.info('Test')
+        super(DDPHandler, self).__init__(*args, **kwargs)
+
+    def emit(self, record):
+        """Emit a formatted log record via DDP."""
+        if getattr(this, 'subs', {}).get('Logs', False):
+            this.send({
+                'msg': ADDED,
+                'collection': 'logs',
+                'id': meteor_random_id('/collection/logs'),
+                'fields': {
+                    # 'name': record.name,
+                    # 'levelno': record.levelno,
+                    'levelname': record.levelname,
+                    # 'pathname': record.pathname,
+                    # 'lineno': record.lineno,
+                    'msg': record.msg,
+                    'args': record.args,
+                    # 'exc_info': record.exc_info,
+                    # 'funcName': record.funcName,
+                },
+            })

dddp/main.py

@@ -60,7 +60,7 @@ def ddpp_sockjs_info(environ, start_response):
     ]))
 
 
-def serve(listen, debug=False, **ssl_args):
+def serve(listen, debug=False, verbosity=1, debug_port=0, **ssl_args):
     """Spawn greenlets for handling websockets and PostgreSQL calls."""
     import signal
     from django.apps import apps
@@ -99,7 +99,7 @@ def serve(listen, debug=False, **ssl_args):
     )
 
     # setup WebSocketServer to dispatch web requests
-    webservers = [
+    servers = [
         geventwebsocket.WebSocketServer(
             (host, port),
             resource,
@@ -113,8 +113,8 @@ def serve(listen, debug=False, **ssl_args):
     def killall(*args, **kwargs):
         """Kill all green threads."""
         pgworker.stop()
-        for webserver in webservers:
-            webserver.stop()
+        for server in servers:
+            server.stop()
 
     # die gracefully with SIGINT or SIGQUIT
     gevent.signal(signal.SIGINT, killall)
@@ -133,19 +133,38 @@ def serve(listen, debug=False, **ssl_args):
     )
 
     # start greenlets
+    if debug_port:
+        from gevent.backdoor import BackdoorServer
+        servers.append(
+            BackdoorServer(
+                ('127.0.0.1', debug_port),
+                banner='Django DDP',
+                locals={
+                    'servers': servers,
+                    'pgworker': pgworker,
+                    'killall': killall,
+                    'api': api,
+                    'resource': resource,
+                    'settings': settings,
+                    'wsgi_app': wsgi_app,
+                    'wsgi_name': wsgi_name,
+                },
+            )
+        )
+
     pgworker.start()
     print('=> Started PostgresGreenlet.')
-    web_threads = [
-        gevent.spawn(webserver.serve_forever)
-        for webserver
-        in webservers
+    threads = [
+        gevent.spawn(server.serve_forever)
+        for server
+        in servers
     ]
     print('=> Started DDPWebSocketApplication.')
     print('=> Started your app (%s).' % wsgi_name)
     print('')
     for host, port in listen:
         print('=> App running at: http://%s:%d/' % (host, port))
-    gevent.joinall(web_threads)
+    gevent.joinall(threads)
     pgworker.stop()
     gevent.joinall([pgworker])
 
@@ -190,6 +209,14 @@ def main():
     import argparse
     parser = argparse.ArgumentParser(description=__doc__)
     django = parser.add_argument_group('Django Options')
+    django.add_argument(
+        '--verbosity', '-v', metavar='VERBOSITY', dest='verbosity', type=int,
+        default=1,
+    )
+    django.add_argument(
+        '--debug-port', metavar='DEBUG_PORT', dest='debug_port', type=int,
+        default=0,
+    )
     django.add_argument(
         '--settings', metavar='SETTINGS', dest='settings',
         help="The Python path to a settings module, e.g. "
@@ -218,8 +245,10 @@ def main():
         os.environ['DJANGO_SETTINGS_MODULE'] = namespace.settings
     serve(
         namespace.listen or [Addr('localhost', 8000)],
+        debug_port=namespace.debug_port,
         keyfile=namespace.keyfile,
         certfile=namespace.certfile,
+        verbosity=namespace.verbosity,
     )
 
 

dddp/postgres.py

@@ -22,6 +22,7 @@ class PostgresGreenlet(gevent.Greenlet):
 
         # queues for processing incoming sub/unsub requests and processing
         self.connections = {}
+        self.chunks = {}
         self._stop_event = gevent.event.Event()
 
         # connect to DB in async mode
@@ -62,7 +63,29 @@ class PostgresGreenlet(gevent.Greenlet):
                         "Got NOTIFY (pid=%d, payload=%r)",
                         notify.pid, notify.payload,
                     )
-                    data = ejson.loads(notify.payload)
+
+                    # read the header and check seq/fin.
+                    hdr, chunk = notify.payload.split('|', 1)
+                    # print('RECEIVE: %s' % hdr)
+                    header = ejson.loads(hdr)
+                    uuid = header['uuid']
+                    size, chunks = self.chunks.setdefault(uuid, [0, {}])
+                    if header['fin']:
+                        size = self.chunks[uuid][0] = header['seq']
+
+                    # stash the chunk
+                    chunks[header['seq']] = chunk
+
+                    if len(chunks) != size:
+                        # haven't got all the chunks yet
+                        continue  # process next NOTIFY in loop
+
+                    # got the last chunk -> process it.
+                    data = ''.join(
+                        chunk for _, chunk in sorted(chunks.items())
+                    )
+                    del self.chunks[uuid]  # don't forget to cleanup!
+                    data = ejson.loads(data)
                     sender = data.pop('_sender', None)
                     tx_id = data.pop('_tx_id', None)
                     for connection_id in data.pop('_connection_ids'):

dddp/server/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'dddp.server.apps.ServerConfig'

dddp/server/apps.py

@@ -1,18 +0,0 @@
-"""Django DDP Server app config."""
-from __future__ import print_function, absolute_import, unicode_literals
-
-import mimetypes
-
-from django.apps import AppConfig
-
-
-class ServerConfig(AppConfig):
-
-    """Django config for dddp.server app."""
-
-    name = 'dddp.server'
-    verbose_name = 'Django DDP Meteor Web Server'
-
-    def ready(self):
-        """Configure Django DDP server app."""
-        mimetypes.init()  # read and process /etc/mime.types

dddp/views.py

@@ -1,9 +1,7 @@
 """Django DDP Server views."""
 from __future__ import print_function, absolute_import, unicode_literals
 
-import io
 import mimetypes
-import os.path
 
 from ejson import dumps, loads
 from django.conf import settings
@@ -63,6 +61,9 @@ class MeteorView(View):
         # super(...).__init__ assigns kwargs to instance.
         super(MeteorView, self).__init__(**kwargs)
 
+        # read and process /etc/mime.types
+        mimetypes.init()
+
         self.url_map = {}
 
         # process `star_json`
@@ -187,8 +188,6 @@ class MeteorView(View):
 
     def get(self, request, path):
         """Return HTML (or other related content) for Meteor."""
-        if path[:1] != '/':
-            path = '/%s' % path
         if path == '/meteor_runtime_config.js':
             config = {
                 'DDP_DEFAULT_CONNECTION_URL': request.build_absolute_uri('/'),

dddp/websocket.py

@@ -106,7 +106,6 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication):
     version = None
     support = None
     connection = None
-    subs = None
     remote_ids = None
     base_handler = BaseHandler()
 
@@ -131,7 +130,7 @@ class DDPWebSocketApplication(geventwebsocket.WebSocketApplication):
             '{0[REMOTE_ADDR]}:{0[REMOTE_PORT]}'.format(
                 self.ws.environ,
             )
-        self.subs = {}
+        this.subs = {}
         self.logger.info('+ %s OPEN', self)
         self.send('o')
         self.send('a["{\\"server_id\\":\\"0\\"}"]')

setup.py

@@ -5,7 +5,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='django-ddp',
-    version='0.10.0',
+    version='0.12.0',
     description=__doc__,
     long_description=open('README.rst').read(),
     author='Tyson Clugg',