commit 06e18f86d0348927c543c7a7d9ed5182b2f1d9b7 Author: Jannis Leidel Date: Tue May 2 13:48:53 2017 +0200 Initial import of the monitors from the old django-celery app. diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..dfbed5d --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,15 @@ +[bumpversion] +current_version = 1.0.0 +commit = True +tag = True +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z]+)? +serialize = + {major}.{minor}.{patch}{releaselevel} + {major}.{minor}.{patch} + +[bumpversion:file:django_celery_monitor/__init__.py] + +[bumpversion:file:docs/includes/introduction.txt] + +[bumpversion:file:README.rst] + diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..25a567d --- /dev/null +++ b/.coveragerc @@ -0,0 +1,11 @@ +[run] +branch = 1 +cover_pylib = 0 +include = *django_celery_monitor/* +omit = t/* + +[report] +omit = + */python?.?/* + */site-packages/* + */pypy/* diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..22fb1f9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +# http://editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true +charset = utf-8 +end_of_line = lf + +[Makefile] +indent_style = tab diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..42bed40 --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +.DS_Store +*.pyc +*$py.class +*~ +.*.sw[pon] +dist/ +*.egg-info +*.egg +*.egg/ +build/ +.build/ +_build/ +pip-log.txt +.directory +erl_crash.dump +*.db +Documentation/ +.tox/ +.ropeproject/ +.project +.pydevproject +.idea/ +.coverage +celery/tests/cover/ +.ve* +cover/ +.vagrant/ +*.sqlite3 +.cache/ +htmlcov/ +coverage.xml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a7561aa --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +language: python +sudo: false +cache: false +python: + - "2.7" + - "3.4" + - "3.5" + - "3.6" +os: + - linux +env: + global: + PYTHONUNBUFFERED=yes +install: travis_retry pip install -U tox-travis +script: tox -v -- -v +after_success: + - .tox/$TRAVIS_PYTHON_VERSION/bin/coverage xml + - .tox/$TRAVIS_PYTHON_VERSION/bin/codecov -e TOXENV diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..d5be671 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,92 @@ +========= + AUTHORS +========= +:order: sorted + +Aaron Ross +Adam Endicott +Alex Stapleton +Alvaro Vega +Andrew Frankel +Andrew Watts +Andrii Kostenko +Anton Novosyolov +Ask Solem +Augusto Becciu +Ben Firshman +Brad Jasper +Brett Gibson +Brian Rosner +Charlie DeTar +Christopher Grebs +Dan LaMotte +Darjus Loktevic +David Fischer +David Ziegler +Diego Andres Sanabria Martin +Dmitriy Krasilnikov +Donald Stufft +Eldon Stegall +Eugene Nagornyi +Felix Berger +Glenn Washburn +Gnrhxni +Greg Taylor +Grégoire Cachet +Hari +Idan Zalzberg +Ionel Maries Cristian +Jannis Leidel +Jason Baker +Jay States +Jeff Balogh +Jeff Fischer +Jeffrey Hu +Jens Alm +Jerzy Kozera +Jesper Noehr +John Andrews +John Watson +Jonas Haag +Jonatan Heyman +Josh Drake +José Moreira +Jude Nagurney +Justin Quick +Keith Perkins +Kirill Panshin +Mark Hellewell +Mark Lavin +Mark Stover +Maxim Bodyansky +Michael Elsdoerfer +Michael van Tellingen +Mikhail Korobov +Olivier Tabone +Patrick Altman +Piotr Bulinski +Piotr Sikora +Reza Lotun +Rockallite Wulf +Roger Barnes +Roman Imankulov +Rune Halvorsen +Sam Cooke +Scott Rubin +Sean Creeley +Serj Zavadsky +Simon Charette +Spencer Ellinor +Theo Spears +Timo Sugliani +Vincent Driessen +Vitaly Babiy +Vladislav Poluhin +Weipin Xia +Wes Turner +Wes Winham +Williams Mendez +WoLpH +dongweiming +zeez diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..446f4fc --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,14 @@ +.. _changelog: + +================ + Change history +================ + +.. _version-1.0.0: + +1.0.0 +===== +:release-date: 2017-05-02 11:25 a.m. UTC+2 +:release-by: Jannis Leidel + +- Initial release by extracting the monitors from the old django-celery app. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3e5f551 --- /dev/null +++ b/LICENSE @@ -0,0 +1,55 @@ +Copyright (c) 2017 Jannis Leidel. All Rights Reserved. +Copyright (c) 2015-2016 Ask Solem. All Rights Reserved. +Copyright (c) 2012-2014 GoPivotal, Inc. All Rights Reserved. +Copyright (c) 2009-2012 Ask Solem. All Rights Reserved. + +django-celery-monitor is licensed under The BSD License (3 Clause, also known as +the new BSD license). The license is an OSI approved Open Source +license and is GPL-compatible(1). + +The license text can also be found here: +http://www.opensource.org/licenses/BSD-3-Clause + +License +======= + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Ask Solem nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Ask Solem OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +Documentation License +===================== + +The documentation portion of django-celery-monitor (the rendered contents of the +"docs" directory of a software distribution or checkout) is supplied +under the "Creative Commons Attribution-ShareAlike 4.0 +International" (CC BY-SA 4.0) License as described by +http://creativecommons.org/licenses/by-sa/4.0/ + +Footnotes +========= +(1) A GPL-compatible license makes it possible to + combine django-celery-monitor with other software that is released + under the GPL, it does not mean that we're distributing + django-celery-monitor under the GPL license. The BSD license, unlike the GPL, + let you distribute a modified version without making your + changes open source. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..b92a3a7 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,17 @@ +include CHANGELOG.rst +include LICENSE +include README.rst +include MANIFEST.in +include setup.cfg +include setup.py +include manage.py +recursive-include docs * +recursive-include extra/* +recursive-include examples * +recursive-include requirements *.txt *.rst +recursive-include t *.py +recursive-include django_celery_monitor *.py + +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] +recursive-exclude * .*.sw* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d647f37 --- /dev/null +++ b/Makefile @@ -0,0 +1,148 @@ +PROJ=django_celery_monitor +PGPIDENT="Celery Security Team" +PYTHON=python +PYTEST=py.test +GIT=git +TOX=tox +ICONV=iconv +FLAKE8=flake8 +FLAKEPLUS=flakeplus +PYDOCSTYLE=pydocstyle +SPHINX2RST=sphinx2rst + +TESTDIR=t +SPHINX_DIR=docs/ +SPHINX_BUILDDIR="${SPHINX_DIR}/_build" +README=README.rst +README_SRC="docs/templates/readme.txt" +CONTRIBUTING=CONTRIBUTING.rst +CONTRIBUTING_SRC="docs/contributing.rst" +SPHINX_HTMLDIR="${SPHINX_BUILDDIR}/html" +DOCUMENTATION=Documentation +FLAKEPLUSTARGET=2.7 + +all: help + +help: + @echo "docs - Build documentation." + @echo "test-all - Run tests for all supported python versions." + @echo "distcheck ---------- - Check distribution for problems." + @echo " test - Run unittests using current python." + @echo " lint ------------ - Check codebase for problems." + @echo " apicheck - Check API reference coverage." + @echo " configcheck - Check configuration reference coverage." + @echo " readmecheck - Check README.rst encoding." + @echo " contribcheck - Check CONTRIBUTING.rst encoding" + @echo " flakes -------- - Check code for syntax and style errors." + @echo " flakecheck - Run flake8 on the source code." + @echo " flakepluscheck - Run flakeplus on the source code." + @echo " pep257check - Run pydocstyle on the source code." + @echo "readme - Regenerate README.rst file." + @echo "contrib - Regenerate CONTRIBUTING.rst file" + @echo "clean-dist --------- - Clean all distribution build artifacts." + @echo " clean-git-force - Remove all uncomitted files." + @echo " clean ------------ - Non-destructive clean" + @echo " clean-pyc - Remove .pyc/__pycache__ files" + @echo " clean-docs - Remove documentation build artifacts." + @echo " clean-build - Remove setup artifacts." + @echo "bump - Bump patch version number." + @echo "bump-minor - Bump minor version number." + @echo "bump-major - Bump major version number." + @echo "release - Make PyPI release." + +clean: clean-docs clean-pyc clean-build + +clean-dist: clean clean-git-force + +bump: + bumpversion patch + +bump-minor: + bumpversion minor + +bump-major: + bumpversion major + +release: + python setup.py register sdist bdist_wheel upload --sign --identity="$(PGPIDENT)" + +Documentation: + (cd "$(SPHINX_DIR)"; $(MAKE) html) + mv "$(SPHINX_HTMLDIR)" $(DOCUMENTATION) + +docs: Documentation + +clean-docs: + -rm -rf "$(SPHINX_BUILDDIR)" + +lint: flakecheck apicheck configcheck readmecheck + +apicheck: + (cd "$(SPHINX_DIR)"; $(MAKE) apicheck) + +configcheck: + true + +flakecheck: + $(FLAKE8) "$(PROJ)" "$(TESTDIR)" + +flakediag: + -$(MAKE) flakecheck + +pep257check: + $(PYDOCSTYLE) "$(PROJ)" + +flakepluscheck: + $(FLAKEPLUS) --$(FLAKEPLUSTARGET) "$(PROJ)" "$(TESTDIR)" + +flakeplusdiag: + -$(MAKE) flakepluscheck + +flakes: flakediag flakeplusdiag pep257check + +clean-readme: + -rm -f $(README) + +readmecheck: + $(ICONV) -f ascii -t ascii $(README) >/dev/null + +$(README): + $(SPHINX2RST) "$(README_SRC)" --ascii > $@ + +readme: clean-readme $(README) readmecheck + +clean-contrib: + -rm -f "$(CONTRIBUTING)" + +$(CONTRIBUTING): + $(SPHINX2RST) "$(CONTRIBUTING_SRC)" > $@ + +contrib: clean-contrib $(CONTRIBUTING) + +clean-pyc: + -find . -type f -a \( -name "*.pyc" -o -name "*$$py.class" \) | xargs rm + -find . -type d -name "__pycache__" | xargs rm -r + +removepyc: clean-pyc + +clean-build: + rm -rf build/ dist/ .eggs/ *.egg-info/ .tox/ .coverage cover/ + +clean-git: + $(GIT) clean -xdn + +clean-git-force: + $(GIT) clean -xdf + +test-all: clean-pyc + $(TOX) + +test: + $(PYTHON) setup.py test + +build: + $(PYTHON) setup.py sdist bdist_wheel + +distcheck: lint test clean + +dist: readme contrib clean-dist build diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..cc9c14f --- /dev/null +++ b/README.rst @@ -0,0 +1,77 @@ +================================================ + Celery Monitor for the Django admin framework. +================================================ + +|build-status| |coverage| |license| |wheel| |pyversion| |pyimp| + +:Version: 1.0.0 +:Web: http://django-celery-monitor.readthedocs.io/ +:Download: http://pypi.python.org/pypi/django-celery-monitor +:Source: http://github.com/jezdez/django-celery-monitor +:Keywords: django, celery, events, monitoring + +About +===== + +This extension enables you to monitor Celery tasks and workers. + +It defines two models (``django_celery_monitor.models.WorkerState`` and +``django_celery_monitor.models.TaskState``) used to store worker and task states +and you can query this database table like any other Django model. +It provides a Camera class (``django_celery_monitor.camera.Camera``) to be +used with the Celery events command line tool to automatically populate the +two models with the current state of the Celery workers and tasks. + +.. _installation: + +Installation +============ + +You can install django-celery-monitor either via the Python Package Index (PyPI) +or from source. + +To install using `pip`,:: + + $ pip install -U django-celery-monitor + +.. _installing-from-source: + +Downloading and installing from source +-------------------------------------- + +Download the latest version of django-celery-monitor from +http://pypi.python.org/pypi/django-celery-monitor + +You can install it by doing the following,:: + + $ tar xvfz django-celery-monitor-0.0.0.tar.gz + $ cd django-celery-monitor-0.0.0 + $ python setup.py build + # python setup.py install + +The last command must be executed as a privileged user if +you are not currently using a virtualenv. + +.. |build-status| image:: https://secure.travis-ci.org/jezdez/django-celery-monitor.svg?branch=master + :alt: Build status + :target: https://travis-ci.org/jezdez/django-celery-monitor + +.. |coverage| image:: https://codecov.io/github/jezdez/django-celery-monitor/coverage.svg?branch=master + :target: https://codecov.io/github/jezdez/django-celery-monitor?branch=master + +.. |license| image:: https://img.shields.io/pypi/l/django-celery-monitor.svg + :alt: BSD License + :target: https://opensource.org/licenses/BSD-3-Clause + +.. |wheel| image:: https://img.shields.io/pypi/wheel/django-celery-monitor.svg + :alt: django-celery-monitor can be installed via wheel + :target: http://pypi.python.org/pypi/django-celery-monitor/ + +.. |pyversion| image:: https://img.shields.io/pypi/pyversions/django-celery-monitor.svg + :alt: Supported Python versions. + :target: http://pypi.python.org/pypi/django-celery-monitor/ + +.. |pyimp| image:: https://img.shields.io/pypi/implementation/django-celery-monitor.svg + :alt: Support Python implementations. + :target: http://pypi.python.org/pypi/django-celery-monitor/ + diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..0a1bddd --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,70 @@ +environment: + + global: + # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the + # /E:ON and /V:ON options are not enabled in the batch script intepreter + # See: http://stackoverflow.com/a/13751649/163740 + WITH_COMPILER: "cmd /E:ON /V:ON /C .\\extra\\appveyor\\run_with_compiler.cmd" + + matrix: + + # Pre-installed Python versions, which Appveyor may upgrade to + # a later point release. + # See: http://www.appveyor.com/docs/installed-software#python + + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7.x" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python34" + PYTHON_VERSION: "3.4.x" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python35" + PYTHON_VERSION: "3.5.x" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python36" + PYTHON_VERSION: "3.6.x" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python27-x64" + PYTHON_VERSION: "2.7.x" + PYTHON_ARCH: "64" + WINDOWS_SDK_VERSION: "v7.0" + + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4.x" + PYTHON_ARCH: "64" + WINDOWS_SDK_VERSION: "v7.1" + + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5.x" + PYTHON_ARCH: "64" + WINDOWS_SDK_VERSION: "v7.1" + + - PYTHON: "C:\\Python36-x64" + PYTHON_VERSION: "3.6.x" + PYTHON_ARCH: "64" + WINDOWS_SDK_VERSION: "v7.1" + +init: + - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" + +install: + - "powershell extra\\appveyor\\install.ps1" + - "%PYTHON%/Scripts/pip.exe install -U setuptools" + +build: off + +test_script: + - "%WITH_COMPILER% %PYTHON%/python setup.py test" + +after_test: + - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" + +artifacts: + - path: dist\* + +#on_success: +# - TODO: upload the content of dist/*.whl to a public wheelhouse diff --git a/django_celery_monitor/__init__.py b/django_celery_monitor/__init__.py new file mode 100644 index 0000000..5a32584 --- /dev/null +++ b/django_celery_monitor/__init__.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +"""Celery monitor for Django.""" +# :copyright: (c) 2016, Ask Solem. +# All rights reserved. +# :license: BSD (3 Clause), see LICENSE for more details. + +from __future__ import absolute_import, unicode_literals + +import re + +from collections import namedtuple + +__version__ = '1.0.0' +__author__ = 'Jannis Leidel' +__contact__ = 'jannis@leidel.info' +__homepage__ = 'https://github.com/jezdez/django-celery-monitor' +__docformat__ = 'restructuredtext' + +# -eof meta- + +version_info_t = namedtuple('version_info_t', ( + 'major', 'minor', 'micro', 'releaselevel', 'serial', +)) + +# bumpversion can only search for {current_version} +# so we have to parse the version here. +_temp = re.match( + r'(\d+)\.(\d+).(\d+)(.+)?', __version__).groups() +VERSION = version_info = version_info_t( + int(_temp[0]), int(_temp[1]), int(_temp[2]), _temp[3] or '', '') +del(_temp) +del(re) + +__all__ = [] + +default_app_config = 'django_celery_monitor.apps.CeleryMonitorConfig' diff --git a/django_celery_monitor/admin.py b/django_celery_monitor/admin.py new file mode 100644 index 0000000..25d8a9e --- /dev/null +++ b/django_celery_monitor/admin.py @@ -0,0 +1,260 @@ +"""Result Task Admin interface.""" +from __future__ import absolute_import, unicode_literals + +from __future__ import absolute_import, unicode_literals + +from django.contrib import admin +from django.contrib.admin import helpers +from django.contrib.admin.views import main as main_views +from django.shortcuts import render_to_response +from django.template import RequestContext +from django.utils.encoding import force_text +from django.utils.html import escape +from django.utils.translation import ugettext_lazy as _ + +from celery import current_app +from celery import states +from celery.task.control import broadcast, revoke, rate_limit +from celery.utils.text import abbrtask + +from .admin_utils import action, display_field, fixedwidth +from .models import TaskState, WorkerState +from .humanize import naturaldate +from .utils import make_aware + + +TASK_STATE_COLORS = {states.SUCCESS: 'green', + states.FAILURE: 'red', + states.REVOKED: 'magenta', + states.STARTED: 'yellow', + states.RETRY: 'orange', + 'RECEIVED': 'blue'} +NODE_STATE_COLORS = {'ONLINE': 'green', + 'OFFLINE': 'gray'} + + +class MonitorList(main_views.ChangeList): + """A custom changelist to set the page title automatically.""" + + def __init__(self, *args, **kwargs): + super(MonitorList, self).__init__(*args, **kwargs) + self.title = self.model_admin.list_page_title + + +@display_field(_('state'), 'state') +def colored_state(task): + """Return the task state colored with HTML/CSS according to its level. + + See ``django_celery_monitor.admin.TASK_STATE_COLORS`` for the colors. + """ + state = escape(task.state) + color = TASK_STATE_COLORS.get(task.state, 'black') + return '{1}'.format(color, state) + + +@display_field(_('state'), 'last_heartbeat') +def node_state(node): + """Return the worker state colored with HTML/CSS according to its level. + + See ``django_celery_monitor.admin.NODE_STATE_COLORS`` for the colors. + """ + state = node.is_alive() and 'ONLINE' or 'OFFLINE' + color = NODE_STATE_COLORS[state] + return '{1}'.format(color, state) + + +@display_field(_('ETA'), 'eta') +def eta(task): + """Return the task ETA as a grey "none" if none is provided.""" + if not task.eta: + return 'none' + return escape(make_aware(task.eta)) + + +@display_field(_('when'), 'tstamp') +def tstamp(task): + """Better timestamp rendering. + + Converts the task timestamp to the local timezone and renders + it as a "natural date" -- a human readable version. + """ + value = make_aware(task.tstamp) + return '
{1}
'.format( + escape(str(value)), escape(naturaldate(value)), + ) + + +@display_field(_('name'), 'name') +def name(task): + """Return the task name and abbreviates it to maximum of 16 characters.""" + short_name = abbrtask(task.name, 16) + return '
{1}
'.format( + escape(task.name), escape(short_name), + ) + + +class ModelMonitor(admin.ModelAdmin): + """Base class for task and worker monitors.""" + + can_add = False + can_delete = False + + def get_changelist(self, request, **kwargs): + """Return the custom change list class we defined above.""" + return MonitorList + + def change_view(self, request, object_id, extra_context=None): + """Make sure the title is set correctly.""" + extra_context = extra_context or {} + extra_context.setdefault('title', self.detail_title) + return super(ModelMonitor, self).change_view( + request, object_id, extra_context=extra_context, + ) + + def has_delete_permission(self, request, obj=None): + """Short-circuiting the permission checks based on class attribute.""" + if not self.can_delete: + return False + return super(ModelMonitor, self).has_delete_permission(request, obj) + + def has_add_permission(self, request): + """Short-circuiting the permission checks based on class attribute.""" + if not self.can_add: + return False + return super(ModelMonitor, self).has_add_permission(request) + + +@admin.register(TaskState) +class TaskMonitor(ModelMonitor): + """The Celery task monitor.""" + + detail_title = _('Task detail') + list_page_title = _('Tasks') + rate_limit_confirmation_template = ( + 'django_celery_monitor/confirm_rate_limit.html' + ) + date_hierarchy = 'tstamp' + fieldsets = ( + (None, { + 'fields': ('state', 'task_id', 'name', 'args', 'kwargs', + 'eta', 'runtime', 'worker', 'tstamp'), + 'classes': ('extrapretty', ), + }), + ('Details', { + 'classes': ('collapse', 'extrapretty'), + 'fields': ('result', 'traceback', 'expires'), + }), + ) + list_display = ( + fixedwidth('task_id', name=_('UUID'), pt=8), + colored_state, + name, + fixedwidth('args', pretty=True), + fixedwidth('kwargs', pretty=True), + eta, + tstamp, + 'worker', + ) + readonly_fields = ( + 'state', 'task_id', 'name', 'args', 'kwargs', + 'eta', 'runtime', 'worker', 'result', 'traceback', + 'expires', 'tstamp', + ) + list_filter = ('state', 'name', 'tstamp', 'eta', 'worker') + search_fields = ('name', 'task_id', 'args', 'kwargs', 'worker__hostname') + actions = ['revoke_tasks', + 'terminate_tasks', + 'kill_tasks', + 'rate_limit_tasks'] + + class Media: + """Just some extra colors.""" + + css = {'all': ('django_celery_monitor/style.css', )} + + @action(_('Revoke selected tasks')) + def revoke_tasks(self, request, queryset): + with current_app.default_connection() as connection: + for state in queryset: + revoke(state.task_id, connection=connection) + + @action(_('Terminate selected tasks')) + def terminate_tasks(self, request, queryset): + with current_app.default_connection() as connection: + for state in queryset: + revoke(state.task_id, connection=connection, terminate=True) + + @action(_('Kill selected tasks')) + def kill_tasks(self, request, queryset): + with current_app.default_connection() as connection: + for state in queryset: + revoke(state.task_id, connection=connection, + terminate=True, signal='KILL') + + @action(_('Rate limit selected tasks')) + def rate_limit_tasks(self, request, queryset): + tasks = set([task.name for task in queryset]) + opts = self.model._meta + app_label = opts.app_label + if request.POST.get('post'): + rate = request.POST['rate_limit'] + with current_app.default_connection() as connection: + for task_name in tasks: + rate_limit(task_name, rate, connection=connection) + return None + + context = { + 'title': _('Rate limit selection'), + 'queryset': queryset, + 'object_name': force_text(opts.verbose_name), + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + 'opts': opts, + 'app_label': app_label, + } + + return render_to_response( + self.rate_limit_confirmation_template, context, + context_instance=RequestContext(request), + ) + + def get_actions(self, request): + actions = super(TaskMonitor, self).get_actions(request) + actions.pop('delete_selected', None) + return actions + + def get_queryset(self, request): + qs = super(TaskMonitor, self).get_queryset(request) + return qs.select_related('worker') + + +@admin.register(WorkerState) +class WorkerMonitor(ModelMonitor): + """The Celery worker monitor.""" + + can_add = True + detail_title = _('Node detail') + list_page_title = _('Worker Nodes') + list_display = ('hostname', node_state) + readonly_fields = ('last_heartbeat', ) + actions = ['shutdown_nodes', + 'enable_events', + 'disable_events'] + + @action(_('Shutdown selected worker nodes')) + def shutdown_nodes(self, request, queryset): + broadcast('shutdown', destination=[n.hostname for n in queryset]) + + @action(_('Enable event mode for selected nodes.')) + def enable_events(self, request, queryset): + broadcast('enable_events', + destination=[n.hostname for n in queryset]) + + @action(_('Disable event mode for selected nodes.')) + def disable_events(self, request, queryset): + broadcast('disable_events', + destination=[n.hostname for n in queryset]) + + def get_actions(self, request): + actions = super(WorkerMonitor, self).get_actions(request) + actions.pop('delete_selected', None) + return actions diff --git a/django_celery_monitor/admin_utils.py b/django_celery_monitor/admin_utils.py new file mode 100644 index 0000000..3cc592e --- /dev/null +++ b/django_celery_monitor/admin_utils.py @@ -0,0 +1,53 @@ +"""Some helpers for the admin monitors.""" +from __future__ import absolute_import, unicode_literals + +from pprint import pformat + +from django.utils.html import escape + +FIXEDWIDTH_STYLE = '''\ +{2} \ +''' + + +def _attrs(**kwargs): + def _inner(fun): + for attr_name, attr_value in kwargs.items(): + setattr(fun, attr_name, attr_value) + return fun + return _inner + + +def display_field(short_description, admin_order_field, + allow_tags=True, **kwargs): + """Set some display_field attributes.""" + return _attrs(short_description=short_description, + admin_order_field=admin_order_field, + allow_tags=allow_tags, **kwargs) + + +def action(short_description, **kwargs): + """Set some admin action attributes.""" + return _attrs(short_description=short_description, **kwargs) + + +def fixedwidth(field, name=None, pt=6, width=16, maxlen=64, pretty=False): + """Render a field with a fixed width.""" + @display_field(name or field, field) + def f(task): + val = getattr(task, field) + if pretty: + val = pformat(val, width=width) + if val.startswith("u'") or val.startswith('u"'): + val = val[2:-1] + shortval = val.replace(',', ',\n') + shortval = shortval.replace('\n', '|br/|') + + if len(shortval) > maxlen: + shortval = shortval[:maxlen] + '...' + styled = FIXEDWIDTH_STYLE.format( + escape(val[:255]), pt, escape(shortval), + ) + return styled.replace('|br/|', '
') + return f diff --git a/django_celery_monitor/apps.py b/django_celery_monitor/apps.py new file mode 100644 index 0000000..b2806bb --- /dev/null +++ b/django_celery_monitor/apps.py @@ -0,0 +1,15 @@ +"""Application configuration.""" +from __future__ import absolute_import, unicode_literals + +from django.apps import AppConfig +from django.utils.translation import ugettext_lazy as _ + +__all__ = ['CeleryMonitorConfig'] + + +class CeleryMonitorConfig(AppConfig): + """Default configuration for the django_celery_monitor app.""" + + name = 'django_celery_monitor' + label = 'celery_monitor' + verbose_name = _('Celery Monitor') diff --git a/django_celery_monitor/camera.py b/django_celery_monitor/camera.py new file mode 100644 index 0000000..905e364 --- /dev/null +++ b/django_celery_monitor/camera.py @@ -0,0 +1,149 @@ +"""The Celery events camera.""" +from __future__ import absolute_import, unicode_literals + +from collections import defaultdict +from datetime import timedelta + +from django.conf import settings + +from celery import states +from celery.events.state import Task +from celery.events.snapshot import Polaroid +from celery.five import monotonic +from celery.utils.log import get_logger +from celery.utils.time import maybe_iso8601 + +from .models import WorkerState, TaskState +from .utils import fromtimestamp, correct_awareness + +WORKER_UPDATE_FREQ = 60 # limit worker timestamp write freq. +SUCCESS_STATES = frozenset([states.SUCCESS]) + +NOT_SAVED_ATTRIBUTES = frozenset(['name', 'args', 'kwargs', 'eta']) + +logger = get_logger(__name__) +debug = logger.debug + + +class Camera(Polaroid): + """The Celery events Polaroid snapshot camera. + + Stores task and worker state in the data models + ``django_celery_monitor.models.TaskState`` and + ``django_celery_monitor.models.WorkerState``. + """ + + TaskState = TaskState + WorkerState = WorkerState + + clear_after = True + worker_update_freq = WORKER_UPDATE_FREQ + + def __init__(self, *args, **kwargs): + super(Camera, self).__init__(*args, **kwargs) + self._last_worker_write = defaultdict(lambda: (None, None)) + # Expiry can be timedelta or None for never expire. + self.app.add_defaults({ + 'monitors_expire_success': timedelta(days=1), + 'monitors_expire_error': timedelta(days=3), + 'monitors_expire_pending': timedelta(days=5), + }) + + @property + def expire_task_states(self): + """A twople of Celery task states and expiration timedeltas.""" + return ( + (SUCCESS_STATES, self.app.conf.monitors_expire_success), + (states.EXCEPTION_STATES, self.app.conf.monitors_expire_error), + (states.UNREADY_STATES, self.app.conf.monitors_expire_pending), + ) + + def get_heartbeat(self, worker): + try: + heartbeat = worker.heartbeats[-1] + except IndexError: + return + return fromtimestamp(heartbeat) + + def handle_worker(self, hostname_worker): + (hostname, worker) = hostname_worker + last_write, obj = self._last_worker_write[hostname] + if (not last_write or + monotonic() - last_write > self.worker_update_freq): + obj, _ = self.WorkerState.objects.update_or_create( + hostname=hostname, + defaults={'last_heartbeat': self.get_heartbeat(worker)}, + ) + self._last_worker_write[hostname] = (monotonic(), obj) + return obj + + def handle_task(self, uuid_task, worker=None): + """Handle snapshotted event.""" + uuid, task = uuid_task + if task.worker and task.worker.hostname: + worker = self.handle_worker( + (task.worker.hostname, task.worker), + ) + + defaults = { + 'name': task.name, + 'args': task.args, + 'kwargs': task.kwargs, + 'eta': correct_awareness(maybe_iso8601(task.eta)), + 'expires': correct_awareness(maybe_iso8601(task.expires)), + 'state': task.state, + 'tstamp': fromtimestamp(task.timestamp), + 'result': task.result or task.exception, + 'traceback': task.traceback, + 'runtime': task.runtime, + 'worker': worker + } + # Some fields are only stored in the RECEIVED event, + # so we should remove these from default values, + # so that they are not overwritten by subsequent states. + [defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES + if defaults[attr] is None] + return self.update_task(task.state, task_id=uuid, defaults=defaults) + + def update_task(self, state, **kwargs): + objects = self.TaskState.objects + defaults = kwargs.pop('defaults', None) or {} + if not defaults.get('name'): + return + obj, created = objects.get_or_create(defaults=defaults, **kwargs) + if created: + return obj + else: + if states.state(state) < states.state(obj.state): + keep = Task.merge_rules[states.RECEIVED] + defaults = dict( + (k, v) for k, v in defaults.items() + if k not in keep + ) + + for k, v in defaults.items(): + setattr(obj, k, v) + obj.save() + + return obj + + def on_shutter(self, state, commit_every=100): + + def _handle_tasks(): + for i, task in enumerate(state.tasks.items()): + self.handle_task(task) + + for worker in state.workers.items(): + self.handle_worker(worker) + _handle_tasks() + + def on_cleanup(self): + expired = (self.TaskState.objects.expire_by_states(states, expires) + for states, expires in self.expire_task_states) + dirty = sum(item for item in expired if item is not None) + if dirty: + debug('Cleanup: Marked %s objects as dirty.', dirty) + self.TaskState.objects.purge() + debug('Cleanup: %s objects purged.', dirty) + return dirty + return 0 diff --git a/django_celery_monitor/humanize.py b/django_celery_monitor/humanize.py new file mode 100644 index 0000000..26a7d34 --- /dev/null +++ b/django_celery_monitor/humanize.py @@ -0,0 +1,84 @@ +"""Some helpers to humanize values.""" +from __future__ import absolute_import, unicode_literals + +from datetime import datetime + +from django.utils.translation import ungettext, ugettext as _ +from .utils import now + + +def pluralize_year(n): + """Return a string with the number of yeargs ago.""" + return ungettext(_('{num} year ago'), _('{num} years ago'), n) + + +def pluralize_month(n): + """Return a string with the number of months ago.""" + return ungettext(_('{num} month ago'), _('{num} months ago'), n) + + +def pluralize_week(n): + """Return a string with the number of weeks ago.""" + return ungettext(_('{num} week ago'), _('{num} weeks ago'), n) + + +def pluralize_day(n): + """Return a string with the number of days ago.""" + return ungettext(_('{num} day ago'), _('{num} days ago'), n) + + +OLDER_CHUNKS = ( + (365.0, pluralize_year), + (30.0, pluralize_month), + (7.0, pluralize_week), + (1.0, pluralize_day), +) + + +def naturaldate(date, include_seconds=False): + """Convert datetime into a human natural date string.""" + if not date: + return '' + + right_now = now() + today = datetime(right_now.year, right_now.month, + right_now.day, tzinfo=right_now.tzinfo) + delta = right_now - date + delta_midnight = today - date + + days = delta.days + hours = delta.seconds // 3600 + minutes = delta.seconds // 60 + seconds = delta.seconds + + if days < 0: + return _('just now') + + if days == 0: + if hours == 0: + if minutes > 0: + return ungettext( + _('{minutes} minute ago'), + _('{minutes} minutes ago'), minutes + ).format(minutes=minutes) + else: + if include_seconds and seconds: + return ungettext( + _('{seconds} second ago'), + _('{seconds} seconds ago'), seconds + ).format(seconds=seconds) + return _('just now') + else: + return ungettext( + _('{hours} hour ago'), _('{hours} hours ago'), hours + ).format(hours=hours) + + if delta_midnight.days == 0: + return _('yesterday at {time}').format(time=date.strftime('%H:%M')) + + count = 0 + for chunk, pluralizefun in OLDER_CHUNKS: + if days >= chunk: + count = int(round((delta_midnight.days + 1) / chunk, 0)) + fmt = pluralizefun(count) + return fmt.format(num=count) diff --git a/django_celery_monitor/managers.py b/django_celery_monitor/managers.py new file mode 100644 index 0000000..5ebd2ba --- /dev/null +++ b/django_celery_monitor/managers.py @@ -0,0 +1,35 @@ +"""The model managers.""" +from __future__ import absolute_import, unicode_literals + +from celery.utils.time import maybe_timedelta +from django.db import connections, models, router, transaction + +from .utils import now + + +class TaskStateManager(models.Manager): + """A custom models manager for the TaskState model with some helpers.""" + + def connection_for_write(self): + return connections[router.db_for_write(self.model)] + + def active(self): + return self.filter(hidden=False) + + def expired(self, states, expires, nowfun=now): + return self.filter(state__in=states, + tstamp__lte=nowfun() - maybe_timedelta(expires)) + + def expire_by_states(self, states, expires): + if expires is not None: + return self.expired(states, expires).update(hidden=True) + + def purge(self): + """Purge all expired events.""" + meta = self.model._meta + with transaction.atomic(): + cursor = self.connection_for_write().cursor() + cursor.execute( + 'DELETE FROM {0.db_table} WHERE hidden=%s'.format(meta), + (True, ), + ) diff --git a/django_celery_monitor/migrations/0001_initial.py b/django_celery_monitor/migrations/0001_initial.py new file mode 100644 index 0000000..6d6a34c --- /dev/null +++ b/django_celery_monitor/migrations/0001_initial.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='TaskState', + fields=[ + ('id', models.AutoField(auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID')), + ('state', models.CharField( + choices=[('FAILURE', 'FAILURE'), + ('PENDING', 'PENDING'), + ('RECEIVED', 'RECEIVED'), + ('RETRY', 'RETRY'), + ('REVOKED', 'REVOKED'), + ('STARTED', 'STARTED'), + ('SUCCESS', 'SUCCESS')], + db_index=True, + max_length=64, + verbose_name='state', + )), + ('task_id', models.CharField( + max_length=36, + unique=True, + verbose_name='UUID', + )), + ('name', models.CharField( + db_index=True, + max_length=200, + null=True, + verbose_name='name', + )), + ('tstamp', models.DateTimeField( + db_index=True, + verbose_name='event received at', + )), + ('args', models.TextField( + null=True, + verbose_name='Arguments', + )), + ('kwargs', models.TextField( + null=True, + verbose_name='Keyword arguments', + )), + ('eta', models.DateTimeField( + null=True, + verbose_name='ETA', + )), + ('expires', models.DateTimeField( + null=True, + verbose_name='expires', + )), + ('result', models.TextField( + null=True, + verbose_name='result', + )), + ('traceback', models.TextField( + null=True, + verbose_name='traceback', + )), + ('runtime', models.FloatField( + help_text='in seconds if task succeeded', + null=True, + verbose_name='execution time', + )), + ('retries', models.IntegerField( + default=0, + verbose_name='number of retries', + )), + ('hidden', models.BooleanField( + db_index=True, + default=False, + editable=False, + )), + ], + options={ + 'ordering': ['-tstamp'], + 'get_latest_by': 'tstamp', + 'verbose_name_plural': 'tasks', + 'verbose_name': 'task', + }, + ), + migrations.CreateModel( + name='WorkerState', + fields=[ + ('id', models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + )), + ('hostname', models.CharField( + max_length=255, + unique=True, + verbose_name='hostname', + )), + ('last_heartbeat', models.DateTimeField( + db_index=True, + null=True, + verbose_name='last heartbeat', + )), + ], + options={ + 'ordering': ['-last_heartbeat'], + 'get_latest_by': 'last_heartbeat', + 'verbose_name_plural': 'workers', + 'verbose_name': 'worker', + }, + ), + migrations.AddField( + model_name='taskstate', + name='worker', + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to='celery_monitor.WorkerState', + verbose_name='worker' + ), + ), + ] diff --git a/django_celery_monitor/migrations/__init__.py b/django_celery_monitor/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_celery_monitor/models.py b/django_celery_monitor/models.py new file mode 100644 index 0000000..954cb33 --- /dev/null +++ b/django_celery_monitor/models.py @@ -0,0 +1,103 @@ +"""The data models for the task and worker states.""" +from __future__ import absolute_import, unicode_literals + +from time import time, mktime, gmtime + +from django.db import models +from django.utils.translation import ugettext_lazy as _ +from django.conf import settings + +from celery import states +from celery.events.state import heartbeat_expires +from celery.five import python_2_unicode_compatible + +from . import managers + +ALL_STATES = sorted(states.ALL_STATES) +TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) + + +@python_2_unicode_compatible +class WorkerState(models.Model): + """The data model to store the worker state in.""" + + hostname = models.CharField(_('hostname'), max_length=255, unique=True) + last_heartbeat = models.DateTimeField(_('last heartbeat'), null=True, + db_index=True) + + class Meta: + """Model meta-data.""" + + verbose_name = _('worker') + verbose_name_plural = _('workers') + get_latest_by = 'last_heartbeat' + ordering = ['-last_heartbeat'] + + def __str__(self): + return self.hostname + + def __repr__(self): + return ''.format(self) + + def is_alive(self): + if self.last_heartbeat: + # Use UTC timestamp if USE_TZ is true, or else use local timestamp + timestamp = mktime(gmtime()) if settings.USE_TZ else time() + return timestamp < heartbeat_expires(self.heartbeat_timestamp) + return False + + @property + def heartbeat_timestamp(self): + return mktime(self.last_heartbeat.timetuple()) + + +@python_2_unicode_compatible +class TaskState(models.Model): + """The data model to store the task state in.""" + + state = models.CharField( + _('state'), max_length=64, choices=TASK_STATE_CHOICES, db_index=True, + ) + task_id = models.CharField(_('UUID'), max_length=36, unique=True) + name = models.CharField( + _('name'), max_length=200, null=True, db_index=True, + ) + tstamp = models.DateTimeField(_('event received at'), db_index=True) + args = models.TextField(_('Arguments'), null=True) + kwargs = models.TextField(_('Keyword arguments'), null=True) + eta = models.DateTimeField(_('ETA'), null=True) + expires = models.DateTimeField(_('expires'), null=True) + result = models.TextField(_('result'), null=True) + traceback = models.TextField(_('traceback'), null=True) + runtime = models.FloatField( + _('execution time'), null=True, + help_text=_('in seconds if task succeeded'), + ) + retries = models.IntegerField(_('number of retries'), default=0) + worker = models.ForeignKey( + WorkerState, null=True, verbose_name=_('worker'), + on_delete=models.CASCADE, + ) + hidden = models.BooleanField(editable=False, default=False, db_index=True) + + objects = managers.TaskStateManager() + + class Meta: + """Model meta-data.""" + + verbose_name = _('task') + verbose_name_plural = _('tasks') + get_latest_by = 'tstamp' + ordering = ['-tstamp'] + + def __str__(self): + name = self.name or 'UNKNOWN' + s = '{0.state:<10} {0.task_id:<36} {1}'.format(self, name) + if self.eta: + s += ' eta:{0.eta}'.format(self) + return s + + def __repr__(self): + return ''.format( + self, self.name or 'UNKNOWN', + ) diff --git a/django_celery_monitor/static/django_celery_monitors/style.css b/django_celery_monitor/static/django_celery_monitors/style.css new file mode 100644 index 0000000..b4f4c6a --- /dev/null +++ b/django_celery_monitor/static/django_celery_monitors/style.css @@ -0,0 +1,4 @@ +.form-row.field-traceback p { + font-family: monospace; + white-space: pre; +} diff --git a/django_celery_monitor/templates/django_celery_monitors/confirm_rate_limit.html b/django_celery_monitor/templates/django_celery_monitors/confirm_rate_limit.html new file mode 100644 index 0000000..6152b76 --- /dev/null +++ b/django_celery_monitor/templates/django_celery_monitors/confirm_rate_limit.html @@ -0,0 +1,25 @@ +{% extends "admin/base_site.html" %} +{% load i18n %} + +{% block breadcrumbs %} +
+ {% trans "Home" %} › + {{ app_label|capfirst }} › + {{ opts.verbose_name_plural|capfirst }} › + {% trans 'Rate limit selected tasks' %} +
+{% endblock %} + +{% block content %} +
{% csrf_token %} +
+ {% for obj in queryset %} + + {% endfor %} + + + + +
+
+{% endblock %} diff --git a/django_celery_monitor/utils.py b/django_celery_monitor/utils.py new file mode 100644 index 0000000..d8d68f3 --- /dev/null +++ b/django_celery_monitor/utils.py @@ -0,0 +1,51 @@ +"""Utilities.""" +# -- XXX This module must not use translation as that causes +# -- a recursive loader import! +from __future__ import absolute_import, unicode_literals + +from datetime import datetime + +from django.conf import settings +from django.utils import timezone + +# see Issue celery/django-celery#222 +now_localtime = getattr(timezone, 'template_localtime', timezone.localtime) + + +def now(): + """Return the current date and time.""" + if getattr(settings, 'USE_TZ', False): + return now_localtime(timezone.now()) + else: + return timezone.now() + + +def make_aware(value): + """Make the given datetime aware of a timezone.""" + if settings.USE_TZ: + # naive datetimes are assumed to be in UTC. + if timezone.is_naive(value): + value = timezone.make_aware(value, timezone.utc) + # then convert to the Django configured timezone. + default_tz = timezone.get_default_timezone() + value = timezone.localtime(value, default_tz) + return value + + +def correct_awareness(value): + """Fix the given datetime timezone awareness.""" + if isinstance(value, datetime): + if settings.USE_TZ: + return make_aware(value) + elif timezone.is_aware(value): + default_tz = timezone.get_default_timezone() + return timezone.make_naive(value, default_tz) + return value + + +def fromtimestamp(value): + """Return an aware or naive datetime from the given timestamp.""" + if settings.USE_TZ: + return make_aware(datetime.utcfromtimestamp(value)) + else: + return datetime.fromtimestamp(value) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d5624b4 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,238 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) + $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " apicheck to verify that all modules are present in autodoc" + @echo " configcheck to verify that all modules are present in autodoc" + @echo " spelling to run a spell checker on the documentation" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PROJ.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PROJ.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/PROJ" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PROJ" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: apicheck +apicheck: + $(SPHINXBUILD) -b apicheck $(ALLSPHINXOPTS) $(BUILDDIR)/apicheck + +.PHONY: configcheck +configcheck: + $(SPHINXBUILD) -b configcheck $(ALLSPHINXOPTS) $(BUILDDIR)/configcheck + +.PHONY: spelling +spelling: + SPELLCHECK=1 $(SPHINXBUILD) -b spelling $(ALLSPHINXOPTS) $(BUILDDIR)/spelling + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/_static/.keep b/docs/_static/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docs/_templates/.keep b/docs/_templates/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000..565b052 --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1 @@ +.. include:: ../CHANGELOG.rst diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..3bd0bf5 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, unicode_literals + +import os + +from sphinx_celery import conf + +globals().update(conf.build_config( + 'django_celery_monitor', __file__, + project='django_celery_monitor', + # version_dev='2.0', + # version_stable='1.4', + canonical_url='http://django-celery-monitor.readthedocs.io', + webdomain='', + github_project='jezdez/django-celery-monitor', + copyright='2009-2017', + django_settings='proj.settings', + include_intersphinx={'python', 'sphinx', 'django', 'celery'}, + path_additions=[os.path.join(os.pardir, 't')], + extra_extensions=['sphinx.ext.napoleon'], + html_logo='images/logo.png', + html_favicon='images/favicon.ico', + html_prepend_sidebars=[], + apicheck_ignore_modules=[ + 'django_celery_monitor', + 'django_celery_monitor.apps', + 'django_celery_monitor.admin', + r'django_celery_monitor.migrations.*', + ], +)) diff --git a/docs/copyright.rst b/docs/copyright.rst new file mode 100644 index 0000000..b1cae98 --- /dev/null +++ b/docs/copyright.rst @@ -0,0 +1,28 @@ +Copyright +========= + +*django-celery-monitor User Manual* + +by Ask Solem + +.. |copy| unicode:: U+000A9 .. COPYRIGHT SIGN + +Copyright |copy| 2016, Ask Solem + +All rights reserved. This material may be copied or distributed only +subject to the terms and conditions set forth in the `Creative Commons +Attribution-ShareAlike 4.0 International` +`_ license. + +You may share and adapt the material, even for commercial purposes, but +you must give the original author credit. +If you alter, transform, or build upon this +work, you may distribute the resulting work only under the same license or +a license compatible to this one. + +.. note:: + + While the django-celery-monitor *documentation* is offered under the + Creative Commons *Attribution-ShareAlike 4.0 International* license + the django-celery-monitor *software* is offered under the + `BSD License (3 Clause) `_ diff --git a/docs/glossary.rst b/docs/glossary.rst new file mode 100644 index 0000000..671a036 --- /dev/null +++ b/docs/glossary.rst @@ -0,0 +1,10 @@ +.. _glossary: + +Glossary +======== + +.. glossary:: + :sorted: + + term + Description of term diff --git a/docs/images/favicon.ico b/docs/images/favicon.ico new file mode 100644 index 0000000..f16149f Binary files /dev/null and b/docs/images/favicon.ico differ diff --git a/docs/images/logo.png b/docs/images/logo.png new file mode 100644 index 0000000..6795fc6 Binary files /dev/null and b/docs/images/logo.png differ diff --git a/docs/includes/installation.txt b/docs/includes/installation.txt new file mode 100644 index 0000000..37eb7e5 --- /dev/null +++ b/docs/includes/installation.txt @@ -0,0 +1,29 @@ +.. _installation: + +Installation +============ + +You can install django-celery-monitor either via the Python Package Index (PyPI) +or from source. + +To install using `pip`,:: + + $ pip install -U django-celery-monitor + +.. _installing-from-source: + +Downloading and installing from source +-------------------------------------- + +Download the latest version of django-celery-monitor from +http://pypi.python.org/pypi/django-celery-monitor + +You can install it by doing the following,:: + + $ tar xvfz django-celery-monitor-0.0.0.tar.gz + $ cd django-celery-monitor-0.0.0 + $ python setup.py build + # python setup.py install + +The last command must be executed as a privileged user if +you are not currently using a virtualenv. diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt new file mode 100644 index 0000000..15e00b6 --- /dev/null +++ b/docs/includes/introduction.txt @@ -0,0 +1,47 @@ +:Version: 1.0.0 +:Web: http://django-celery-monitor.readthedocs.io/ +:Download: http://pypi.python.org/pypi/django-celery-monitor +:Source: http://github.com/jezdez/django-celery-monitor +:Keywords: django, celery, events, monitoring + +About +===== + +This extension enables you to monitor Celery tasks and workers. + +It defines two models (``django_celery_monitor.models.WorkerState`` and +``django_celery_monitor.models.TaskState``) used to store worker and task states +and you can query this database table like any other Django model. +It provides a Camera class (``django_celery_monitor.camera.Camera``) to be +used with the Celery events command line tool to automatically populate the +two models with the current state of the Celery workers and tasks. + +Configuration +============= + +There are a few settings that regulate how long the task monitor should keep +state entries in the database. Either of the three should be a +``datetime.timedelta`` value or ``None``. + +- ``monitor_task_success_expires`` -- Defaults to ``timedelta(days=1)`` (1 day) + + The period of time to retain monitoring information about tasks with a + ``SUCCESS`` result. + +- ``monitor_task_error_expires`` -- Defaults to ``timedelta(days=3)`` (3 days) + + The period of time to retain monitoring information about tasks with an + errornous result (one of the following event states: ``RETRY``, ``FAILURE``, + ``REVOKED``. + +- ``monitor_task_pending_expires`` -- Defaults to ``timedelta(days=5)`` (5 days) + + The period of time to retain monitoring information about tasks with a + pending result (one of the following event states: ``PENDING``, ``RECEIVED``, + ``STARTED``, ``REJECTED``, ``RETRY``. + +In your Celery configuration simply set them to override the defaults, e.g.:: + + from datetime import timedelta + + monitor_task_success_expires = timedelta(days=7) diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..c5c1bc7 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,32 @@ +====================================================== + django-celery-monitor - Celery Monitoring for Django +====================================================== + +.. include:: includes/introduction.txt + +Contents +======== + +.. toctree:: + :maxdepth: 1 + + copyright + +.. toctree:: + :maxdepth: 2 + + reference/index + +.. toctree:: + :maxdepth: 1 + + changelog + glossary + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..001e23f --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,272 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. epub3 to make an epub3 + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 1>NUL 2>NUL +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PROJ.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PROJ.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "epub3" ( + %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/docs/reference/django_celery_monitor.admin_utils.rst b/docs/reference/django_celery_monitor.admin_utils.rst new file mode 100644 index 0000000..4ecc43e --- /dev/null +++ b/docs/reference/django_celery_monitor.admin_utils.rst @@ -0,0 +1,11 @@ +======================================= + ``django_celery_monitor.admin_utils`` +======================================= + +.. contents:: + :local: +.. currentmodule:: django_celery_monitor.admin_utils + +.. automodule:: django_celery_monitor.admin_utils + :members: + :undoc-members: diff --git a/docs/reference/django_celery_monitor.camera.rst b/docs/reference/django_celery_monitor.camera.rst new file mode 100644 index 0000000..308a40f --- /dev/null +++ b/docs/reference/django_celery_monitor.camera.rst @@ -0,0 +1,11 @@ +================================== + ``django_celery_monitor.camera`` +================================== + +.. contents:: + :local: +.. currentmodule:: django_celery_monitor.camera + +.. automodule:: django_celery_monitor.camera + :members: + :undoc-members: diff --git a/docs/reference/django_celery_monitor.humanize.rst b/docs/reference/django_celery_monitor.humanize.rst new file mode 100644 index 0000000..f469711 --- /dev/null +++ b/docs/reference/django_celery_monitor.humanize.rst @@ -0,0 +1,11 @@ +===================================== + ``django_celery_monitor.humanize`` +===================================== + +.. contents:: + :local: +.. currentmodule:: django_celery_monitor.humanize + +.. automodule:: django_celery_monitor.humanize + :members: + :undoc-members: diff --git a/docs/reference/django_celery_monitor.managers.rst b/docs/reference/django_celery_monitor.managers.rst new file mode 100644 index 0000000..ca141f0 --- /dev/null +++ b/docs/reference/django_celery_monitor.managers.rst @@ -0,0 +1,11 @@ +===================================== + ``django_celery_monitor.managers`` +===================================== + +.. contents:: + :local: +.. currentmodule:: django_celery_monitor.managers + +.. automodule:: django_celery_monitor.managers + :members: + :undoc-members: diff --git a/docs/reference/django_celery_monitor.models.rst b/docs/reference/django_celery_monitor.models.rst new file mode 100644 index 0000000..b2359a1 --- /dev/null +++ b/docs/reference/django_celery_monitor.models.rst @@ -0,0 +1,11 @@ +=================================== + ``django_celery_monitor.models`` +=================================== + +.. contents:: + :local: +.. currentmodule:: django_celery_monitor.models + +.. automodule:: django_celery_monitor.models + :members: + :undoc-members: diff --git a/docs/reference/django_celery_monitor.utils.rst b/docs/reference/django_celery_monitor.utils.rst new file mode 100644 index 0000000..d43d462 --- /dev/null +++ b/docs/reference/django_celery_monitor.utils.rst @@ -0,0 +1,11 @@ +================================== + ``django_celery_monitor.utils`` +================================== + +.. contents:: + :local: +.. currentmodule:: django_celery_monitor.utils + +.. automodule:: django_celery_monitor.utils + :members: + :undoc-members: diff --git a/docs/reference/index.rst b/docs/reference/index.rst new file mode 100644 index 0000000..61cdcfc --- /dev/null +++ b/docs/reference/index.rst @@ -0,0 +1,18 @@ +.. _apiref: + +=============== + API Reference +=============== + +:Release: |version| +:Date: |today| + +.. toctree:: + :maxdepth: 1 + + django_celery_monitor.admin_utils + django_celery_monitor.camera + django_celery_monitor.humanize + django_celery_monitor.managers + django_celery_monitor.models + django_celery_monitor.utils diff --git a/docs/templates/readme.txt b/docs/templates/readme.txt new file mode 100644 index 0000000..d1de117 --- /dev/null +++ b/docs/templates/readme.txt @@ -0,0 +1,32 @@ +================================================ + Celery Monitor for the Django admin framework. +================================================ + +|build-status| |coverage| |license| |wheel| |pyversion| |pyimp| + +.. include:: ../includes/introduction.txt + +.. include:: ../includes/installation.txt + +.. |build-status| image:: https://secure.travis-ci.org/jezdez/django-celery-monitor.svg?branch=master + :alt: Build status + :target: https://travis-ci.org/jezdez/django-celery-monitor + +.. |coverage| image:: https://codecov.io/github/jezdez/django-celery-monitor/coverage.svg?branch=master + :target: https://codecov.io/github/jezdez/django-celery-monitor?branch=master + +.. |license| image:: https://img.shields.io/pypi/l/django-celery-monitor.svg + :alt: BSD License + :target: https://opensource.org/licenses/BSD-3-Clause + +.. |wheel| image:: https://img.shields.io/pypi/wheel/django-celery-monitor.svg + :alt: django-celery-monitor can be installed via wheel + :target: http://pypi.python.org/pypi/django-celery-monitor/ + +.. |pyversion| image:: https://img.shields.io/pypi/pyversions/django-celery-monitor.svg + :alt: Supported Python versions. + :target: http://pypi.python.org/pypi/django-celery-monitor/ + +.. |pyimp| image:: https://img.shields.io/pypi/implementation/django-celery-monitor.svg + :alt: Support Python implementations. + :target: http://pypi.python.org/pypi/django-celery-monitor/ diff --git a/extra/appveyor/install.ps1 b/extra/appveyor/install.ps1 new file mode 100644 index 0000000..3f05628 --- /dev/null +++ b/extra/appveyor/install.ps1 @@ -0,0 +1,85 @@ +# Sample script to install Python and pip under Windows +# Authors: Olivier Grisel and Kyle Kastner +# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ + +$BASE_URL = "https://www.python.org/ftp/python/" +$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" +$GET_PIP_PATH = "C:\get-pip.py" + + +function DownloadPython ($python_version, $platform_suffix) { + $webclient = New-Object System.Net.WebClient + $filename = "python-" + $python_version + $platform_suffix + ".msi" + $url = $BASE_URL + $python_version + "/" + $filename + + $basedir = $pwd.Path + "\" + $filepath = $basedir + $filename + if (Test-Path $filename) { + Write-Host "Reusing" $filepath + return $filepath + } + + # Download and retry up to 5 times in case of network transient errors. + Write-Host "Downloading" $filename "from" $url + $retry_attempts = 3 + for($i=0; $i -lt $retry_attempts; $i++){ + try { + $webclient.DownloadFile($url, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + Write-Host "File saved at" $filepath + return $filepath +} + + +function InstallPython ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "32") { + $platform_suffix = "" + } else { + $platform_suffix = ".amd64" + } + $filepath = DownloadPython $python_version $platform_suffix + Write-Host "Installing" $filepath "to" $python_home + $args = "/qn /i $filepath TARGETDIR=$python_home" + Write-Host "msiexec.exe" $args + Start-Process -FilePath "msiexec.exe" -ArgumentList $args -Wait -Passthru + Write-Host "Python $python_version ($architecture) installation complete" + return $true +} + + +function InstallPip ($python_home) { + $pip_path = $python_home + "/Scripts/pip.exe" + $python_path = $python_home + "/python.exe" + if (-not(Test-Path $pip_path)) { + Write-Host "Installing pip..." + $webclient = New-Object System.Net.WebClient + $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) + Write-Host "Executing:" $python_path $GET_PIP_PATH + Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru + } else { + Write-Host "pip already installed." + } +} + +function InstallPackage ($python_home, $pkg) { + $pip_path = $python_home + "/Scripts/pip.exe" + & $pip_path install $pkg +} + +function main () { + InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON + InstallPip $env:PYTHON + InstallPackage $env:PYTHON wheel +} + +main diff --git a/extra/appveyor/run_with_compiler.cmd b/extra/appveyor/run_with_compiler.cmd new file mode 100644 index 0000000..3a472bc --- /dev/null +++ b/extra/appveyor/run_with_compiler.cmd @@ -0,0 +1,47 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds do not require specific environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows + +SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" +IF %MAJOR_PYTHON_VERSION% == "2" ( + SET WINDOWS_SDK_VERSION="v7.0" +) ELSE IF %MAJOR_PYTHON_VERSION% == "3" ( + SET WINDOWS_SDK_VERSION="v7.1" +) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 +) + +IF "%PYTHON_ARCH%"=="64" ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..43ca0db --- /dev/null +++ b/manage.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +from __future__ import absolute_import, unicode_literals +import os +import sys + +if __name__ == '__main__': + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 't.proj.settings') + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/requirements/default.txt b/requirements/default.txt new file mode 100644 index 0000000..01be9da --- /dev/null +++ b/requirements/default.txt @@ -0,0 +1 @@ +celery>=4.0,<5.0 diff --git a/requirements/docs.txt b/requirements/docs.txt new file mode 100644 index 0000000..f581c15 --- /dev/null +++ b/requirements/docs.txt @@ -0,0 +1,2 @@ +sphinx_celery>=1.1 +Django>=1.10 diff --git a/requirements/pkgutils.txt b/requirements/pkgutils.txt new file mode 100644 index 0000000..c08306b --- /dev/null +++ b/requirements/pkgutils.txt @@ -0,0 +1,8 @@ +setuptools>=20.6.7 +wheel>=0.29.0 +flake8>=2.5.4 +flakeplus>=1.1 +tox>=2.3.1 +sphinx2rst>=1.0 +bumpversion +pydocstyle diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt new file mode 100644 index 0000000..0a1cbfc --- /dev/null +++ b/requirements/test-ci.txt @@ -0,0 +1,2 @@ +pytest-cov +codecov diff --git a/requirements/test-django110.txt b/requirements/test-django110.txt new file mode 100644 index 0000000..0b54502 --- /dev/null +++ b/requirements/test-django110.txt @@ -0,0 +1 @@ +django>=1.10,<1.11 diff --git a/requirements/test-django111.txt b/requirements/test-django111.txt new file mode 100644 index 0000000..5e1444a --- /dev/null +++ b/requirements/test-django111.txt @@ -0,0 +1 @@ +django>=1.11,<2 diff --git a/requirements/test-django18.txt b/requirements/test-django18.txt new file mode 100644 index 0000000..52b42d2 --- /dev/null +++ b/requirements/test-django18.txt @@ -0,0 +1 @@ +django>=1.8,<1.9 diff --git a/requirements/test-django19.txt b/requirements/test-django19.txt new file mode 100644 index 0000000..194e6fb --- /dev/null +++ b/requirements/test-django19.txt @@ -0,0 +1 @@ +django>=1.9,<1.10 diff --git a/requirements/test.txt b/requirements/test.txt new file mode 100644 index 0000000..770bbd3 --- /dev/null +++ b/requirements/test.txt @@ -0,0 +1,4 @@ +case>=1.3.1 +pytest>=3.0 +pytest-django +pytz>dev diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5cbb725 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,16 @@ +[tool:pytest] +testpaths = t/unit +python_classes = test_* +DJANGO_SETTINGS_MODULE=t.proj.settings + +[flake8] +# classes can be lowercase, arguments and variables can be uppercase +# whenever it makes the code more readable. +ignore = N806, N802, N801, N803 + +[pep257] +ignore = D102,D104,D203,D105,D213 +match-dir = [^migrations] + +[wheel] +universal = 1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4d1913d --- /dev/null +++ b/setup.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import re +import sys +import codecs + +import setuptools +import setuptools.command.test + +try: + import platform + _pyimp = platform.python_implementation +except (AttributeError, ImportError): + def _pyimp(): + return 'Python' + +NAME = 'django_celery_monitor' + +E_UNSUPPORTED_PYTHON = '%s 1.0 requires %%s %%s or later!' % (NAME,) + +PYIMP = _pyimp() +PY26_OR_LESS = sys.version_info < (2, 7) +PY3 = sys.version_info[0] == 3 +PY33_OR_LESS = PY3 and sys.version_info < (3, 4) +PYPY_VERSION = getattr(sys, 'pypy_version_info', None) +PYPY = PYPY_VERSION is not None +PYPY24_ATLEAST = PYPY_VERSION and PYPY_VERSION >= (2, 4) + +if PY26_OR_LESS: + raise Exception(E_UNSUPPORTED_PYTHON % (PYIMP, '2.7')) +elif PY33_OR_LESS and not PYPY24_ATLEAST: + raise Exception(E_UNSUPPORTED_PYTHON % (PYIMP, '3.4')) + +# -*- Classifiers -*- + +classes = """ + Development Status :: 5 - Production/Stable + License :: OSI Approved :: BSD License + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy + Framework :: Django + Framework :: Django :: 1.8 + Framework :: Django :: 1.9 + Framework :: Django :: 1.10 + Framework :: Django :: 1.11 + Operating System :: OS Independent + Topic :: Communications + Topic :: System :: Distributed Computing + Topic :: Software Development :: Libraries :: Python Modules +""" +classifiers = [s.strip() for s in classes.split('\n') if s] + +# -*- Distribution Meta -*- + +re_meta = re.compile(r'__(\w+?)__\s*=\s*(.*)') +re_doc = re.compile(r'^"""(.+?)"""') + + +def add_default(m): + attr_name, attr_value = m.groups() + return ((attr_name, attr_value.strip("\"'")),) + + +def add_doc(m): + return (('doc', m.groups()[0]),) + +pats = {re_meta: add_default, + re_doc: add_doc} +here = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(here, NAME, '__init__.py')) as meta_fh: + meta = {} + for line in meta_fh: + if line.strip() == '# -eof meta-': + break + for pattern, handler in pats.items(): + m = pattern.match(line.strip()) + if m: + meta.update(handler(m)) + +# -*- Installation Requires -*- + + +def strip_comments(l): + return l.split('#', 1)[0].strip() + + +def _pip_requirement(req): + if req.startswith('-r '): + _, path = req.split() + return reqs(*path.split('/')) + return [req] + + +def _reqs(*f): + return [ + _pip_requirement(r) for r in ( + strip_comments(l) for l in open( + os.path.join(os.getcwd(), 'requirements', *f)).readlines() + ) if r] + + +def reqs(*f): + return [req for subreq in _reqs(*f) for req in subreq] + +# -*- Long Description -*- + +if os.path.exists('README.rst'): + long_description = codecs.open('README.rst', 'r', 'utf-8').read() +else: + long_description = 'See http://pypi.python.org/pypi/%s' % (NAME,) + +# -*- %%% -*- + + +class pytest(setuptools.command.test.test): + user_options = [('pytest-args=', 'a', 'Arguments to pass to py.test')] + + def initialize_options(self): + setuptools.command.test.test.initialize_options(self) + self.pytest_args = [] + + def run_tests(self): + import pytest + sys.exit(pytest.main(self.pytest_args)) + + +setuptools.setup( + name=NAME, + packages=setuptools.find_packages(exclude=['t', 't.*']), + version=meta['version'], + description=meta['doc'], + long_description=long_description, + keywords='celery django events monitoring', + author=meta['author'], + author_email=meta['contact'], + url=meta['homepage'], + platforms=['any'], + license='BSD', + classifiers=classifiers, + install_requires=reqs('default.txt'), + tests_require=reqs('test.txt'), + cmdclass={'test': pytest}, + zip_safe=False, + include_package_data=False, +) diff --git a/t/__init__.py b/t/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/t/proj/__init__.py b/t/proj/__init__.py new file mode 100644 index 0000000..0a8d3da --- /dev/null +++ b/t/proj/__init__.py @@ -0,0 +1 @@ +from .celery import app as celery_app # noqa diff --git a/t/proj/celery.py b/t/proj/celery.py new file mode 100644 index 0000000..8324e6f --- /dev/null +++ b/t/proj/celery.py @@ -0,0 +1,15 @@ +from __future__ import absolute_import, unicode_literals + +import os + +from celery import Celery + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') + +app = Celery('proj') + +# Using a string here means the worker doesn't have to serialize +# the configuration object. +app.config_from_object('django.conf:settings', namespace='CELERY') + +app.autodiscover_tasks() diff --git a/t/proj/settings.py b/t/proj/settings.py new file mode 100644 index 0000000..068d20b --- /dev/null +++ b/t/proj/settings.py @@ -0,0 +1,117 @@ +""" +Django settings for Test project. + +Generated by 'django-admin startproject' using Django 1.9.1. + +For more information on this file, see +https://docs.djangoproject.com/en/1.9/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.9/ref/settings/ +""" +from __future__ import absolute_import, unicode_literals + +import os +import sys + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +sys.path.insert(0, os.path.abspath(os.path.join(BASE_DIR, os.pardir))) + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'u($kbs9$irs0)436gbo9%!b&#zyd&70tx!n7!i&fl6qun@z1_l' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django_celery_monitor', +] + +MIDDLEWARE_CLASSES = [ +] + +ROOT_URLCONF = 'proj.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'proj.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.9/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'OPTIONS': { + 'timeout': 1000, + }, + } +} + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, + 'dummy': { + 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + }, +} + +# Password validation +# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators + +django_auth = 'django.contrib.auth.password_validation.' + +AUTH_PASSWORD_VALIDATORS = [ +] + + +# Internationalization +# https://docs.djangoproject.com/en/1.9/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.9/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/t/proj/urls.py b/t/proj/urls.py new file mode 100644 index 0000000..ca6c9e2 --- /dev/null +++ b/t/proj/urls.py @@ -0,0 +1,8 @@ +from __future__ import absolute_import, unicode_literals + +from django.conf.urls import url +from django.contrib import admin + +urlpatterns = [ + url(r'^admin/', admin.site.urls), +] diff --git a/t/proj/wsgi.py b/t/proj/wsgi.py new file mode 100644 index 0000000..1170b2b --- /dev/null +++ b/t/proj/wsgi.py @@ -0,0 +1,17 @@ +""" +WSGI config for Test project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ +""" +from __future__ import absolute_import, unicode_literals + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "proj.settings") + +application = get_wsgi_application() diff --git a/t/unit/__init__.py b/t/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/t/unit/conftest.py b/t/unit/conftest.py new file mode 100644 index 0000000..33fca83 --- /dev/null +++ b/t/unit/conftest.py @@ -0,0 +1,36 @@ +from __future__ import absolute_import, unicode_literals + +import pytest + +from celery.contrib.pytest import depends_on_current_app +from celery.contrib.testing.app import TestApp, Trap + +__all__ = ['app', 'depends_on_current_app'] + + +@pytest.fixture(scope='session', autouse=True) +def setup_default_app_trap(): + from celery._state import set_default_app + set_default_app(Trap()) + + +@pytest.fixture() +def app(celery_app): + return celery_app + + +@pytest.fixture(autouse=True) +def test_cases_shortcuts(request, app, patching): + if request.instance: + @app.task + def add(x, y): + return x + y + + # IMPORTANT: We set an .app attribute for every test case class. + request.instance.app = app + request.instance.Celery = TestApp + request.instance.add = add + request.instance.patching = patching + yield + if request.instance: + request.instance.app = None diff --git a/t/unit/test_camera.py b/t/unit/test_camera.py new file mode 100644 index 0000000..95534d1 --- /dev/null +++ b/t/unit/test_camera.py @@ -0,0 +1,256 @@ +from __future__ import absolute_import, unicode_literals + +from datetime import datetime +from itertools import count +from time import time + +import pytest + +from celery import states +from celery.events import Event as _Event +from celery.events.state import State, Worker, Task +from celery.utils import gen_unique_id + +from django.test.utils import override_settings +from django.utils import timezone + +from django_celery_monitor import camera, models +from django_celery_monitor.utils import make_aware, now + + +_ids = count(0) +_clock = count(1) + + +def Event(*args, **kwargs): + kwargs.setdefault('clock', next(_clock)) + kwargs.setdefault('local_received', time()) + return _Event(*args, **kwargs) + + +@pytest.mark.django_db() +@pytest.mark.usefixtures('depends_on_current_app') +class test_Camera: + Camera = camera.Camera + + def create_task(self, worker, **kwargs): + d = dict(uuid=gen_unique_id(), + name='django_celery_monitor.test.task{0}'.format(next(_ids)), + worker=worker) + return Task(**dict(d, **kwargs)) + + @pytest.fixture(autouse=True) + def setup_app(self, app): + self.app = app + self.state = State() + self.cam = self.Camera(self.state) + + def test_constructor(self): + cam = self.Camera(State()) + assert cam.state + assert cam.freq + assert cam.cleanup_freq + assert cam.logger + + def test_get_heartbeat(self): + worker = Worker(hostname='fuzzie') + assert self.cam.get_heartbeat(worker) is None + t1 = time() + t2 = time() + t3 = time() + for t in t1, t2, t3: + worker.event('heartbeat', t, t, {}) + self.state.workers[worker.hostname] = worker + assert ( + self.cam.get_heartbeat(worker) == + make_aware(datetime.fromtimestamp(t3)) + ) + + def test_handle_worker(self): + worker = Worker(hostname='fuzzie') + worker.event('online', time(), time(), {}) + self.cam._last_worker_write.clear() + m = self.cam.handle_worker((worker.hostname, worker)) + assert m + assert m.hostname + assert m.last_heartbeat + assert m.is_alive() + assert str(m) == str(m.hostname) + assert repr(m) + + def test_handle_task_received(self): + worker = Worker(hostname='fuzzie') + worker.event('online', time(), time(), {}) + self.cam.handle_worker((worker.hostname, worker)) + + task = self.create_task(worker) + task.event('received', time(), time(), {}) + assert task.state == states.RECEIVED + mt = self.cam.handle_task((task.uuid, task)) + assert mt.name == task.name + assert str(mt) + assert repr(mt) + mt.eta = now() + assert 'eta' in str(mt) + assert mt in models.TaskState.objects.active() + + def test_handle_task(self): + worker1 = Worker(hostname='fuzzie') + worker1.event('online', time(), time(), {}) + mw = self.cam.handle_worker((worker1.hostname, worker1)) + task1 = self.create_task(worker1) + task1.event('received', time(), time(), {}) + mt = self.cam.handle_task((task1.uuid, task1)) + assert mt.worker == mw + + worker2 = Worker(hostname=None) + task2 = self.create_task(worker2) + task2.event('received', time(), time(), {}) + mt = self.cam.handle_task((task2.uuid, task2)) + assert mt.worker is None + + task1.event('succeeded', time(), time(), {'result': 42}) + assert task1.state == states.SUCCESS + assert task1.result == 42 + mt = self.cam.handle_task((task1.uuid, task1)) + assert mt.name == task1.name + assert mt.result == 42 + + task3 = self.create_task(worker1, name=None) + task3.event('revoked', time(), time(), {}) + mt = self.cam.handle_task((task3.uuid, task3)) + assert mt is None + + def test_handle_task_timezone(self): + worker = Worker(hostname='fuzzie') + worker.event('online', time(), time(), {}) + self.cam.handle_worker((worker.hostname, worker)) + + tstamp = 1464793200.0 # 2016-06-01T15:00:00Z + + with override_settings(USE_TZ=True, TIME_ZONE='Europe/Helsinki'): + task = self.create_task( + worker, + eta='2016-06-01T15:16:17.654321+00:00', + expires='2016-07-01T15:16:17.765432+03:00', + ) + task.event('received', tstamp, tstamp, {}) + mt = self.cam.handle_task((task.uuid, task)) + assert ( + mt.tstamp == + datetime(2016, 6, 1, 15, 0, 0, tzinfo=timezone.utc) + ) + assert ( + mt.eta == + datetime(2016, 6, 1, 15, 16, 17, 654321, tzinfo=timezone.utc) + ) + assert ( + mt.expires == + datetime(2016, 7, 1, 12, 16, 17, 765432, tzinfo=timezone.utc) + ) + + task = self.create_task(worker, eta='2016-06-04T15:16:17.654321') + task.event('received', tstamp, tstamp, {}) + mt = self.cam.handle_task((task.uuid, task)) + assert ( + mt.eta == + datetime(2016, 6, 4, 15, 16, 17, 654321, tzinfo=timezone.utc) + ) + + with override_settings(USE_TZ=False, TIME_ZONE='Europe/Helsinki'): + task = self.create_task( + worker, + eta='2016-06-01T15:16:17.654321+00:00', + expires='2016-07-01T15:16:17.765432+03:00', + ) + task.event('received', tstamp, tstamp, {}) + mt = self.cam.handle_task((task.uuid, task)) + assert mt.tstamp == datetime(2016, 6, 1, 18, 0, 0) + assert mt.eta == datetime(2016, 6, 1, 18, 16, 17, 654321) + assert mt.expires == datetime(2016, 7, 1, 15, 16, 17, 765432) + + task = self.create_task(worker, eta='2016-06-04T15:16:17.654321') + task.event('received', tstamp, tstamp, {}) + mt = self.cam.handle_task((task.uuid, task)) + assert mt.eta == datetime(2016, 6, 4, 15, 16, 17, 654321) + + def assert_expires(self, dec, expired, tasks=10): + # Cleanup leftovers from previous tests + self.cam.on_cleanup() + + worker = Worker(hostname='fuzzie') + worker.event('online', time(), time(), {}) + for total in range(tasks): + task = self.create_task(worker) + task.event('received', time() - dec, time() - dec, {}) + task.event('succeeded', time() - dec, time() - dec, {'result': 42}) + assert task.name + assert self.cam.handle_task((task.uuid, task)) + assert self.cam.on_cleanup() == expired + + def test_on_cleanup_expires(self, dec=332000): + self.assert_expires(dec, 10) + + def test_on_cleanup_does_not_expire_new(self, dec=0): + self.assert_expires(dec, 0) + + def test_on_shutter(self): + state = self.state + cam = self.cam + + ws = ['worker1.ex.com', 'worker2.ex.com', 'worker3.ex.com'] + uus = [gen_unique_id() for i in range(50)] + + events = [Event('worker-online', hostname=ws[0]), + Event('worker-online', hostname=ws[1]), + Event('worker-online', hostname=ws[2]), + Event('task-received', + uuid=uus[0], name='A', hostname=ws[0]), + Event('task-started', + uuid=uus[0], name='A', hostname=ws[0]), + Event('task-received', + uuid=uus[1], name='B', hostname=ws[1]), + Event('task-revoked', + uuid=uus[2], name='C', hostname=ws[2])] + + for event in events: + event['local_received'] = time() + state.event(event) + cam.on_shutter(state) + + for host in ws: + worker = models.WorkerState.objects.get(hostname=host) + assert worker.is_alive() + + t1 = models.TaskState.objects.get(task_id=uus[0]) + assert t1.state == states.STARTED + assert t1.name == 'A' + t2 = models.TaskState.objects.get(task_id=uus[1]) + assert t2.state == states.RECEIVED + t3 = models.TaskState.objects.get(task_id=uus[2]) + assert t3.state == states.REVOKED + + events = [Event('task-succeeded', + uuid=uus[0], hostname=ws[0], result=42), + Event('task-failed', + uuid=uus[1], exception="KeyError('foo')", + hostname=ws[1]), + Event('worker-offline', hostname=ws[0])] + list(map(state.event, events)) + cam._last_worker_write.clear() + cam.on_shutter(state) + + w1 = models.WorkerState.objects.get(hostname=ws[0]) + assert not w1.is_alive() + + t1 = models.TaskState.objects.get(task_id=uus[0]) + assert t1.state == states.SUCCESS + assert t1.result == '42' + assert t1.worker == w1 + + t2 = models.TaskState.objects.get(task_id=uus[1]) + assert t2.state == states.FAILURE + assert t2.result == "KeyError('foo')" + assert t2.worker.hostname == ws[1] + + cam.on_shutter(state) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..712c054 --- /dev/null +++ b/tox.ini @@ -0,0 +1,55 @@ +[tox] +envlist = + py{py,27,34,35}-dj{18,19,110} + py36-dj111 + flake8 + flakeplus + apicheck + pydocstyle + +[travis] +python = + 2.7: py27, flake8, flakeplus, apicheck, pydocstyle + 3.4: py34 + 3.5: py35 + 3.6: py36 + +[testenv] +deps= + -r{toxinidir}/requirements/default.txt + -r{toxinidir}/requirements/test.txt + -r{toxinidir}/requirements/test-ci.txt + + dj111: -r{toxinidir}/requirements/test-django111.txt + dj110: -r{toxinidir}/requirements/test-django110.txt + dj19: -r{toxinidir}/requirements/test-django19.txt + dj18: -r{toxinidir}/requirements/test-django18.txt + + linkcheck,apicheck: -r{toxinidir}/requirements/docs.txt + flake8,flakeplus,pydocstyle: -r{toxinidir}/requirements/pkgutils.txt +sitepackages = False +commands = + py.test -xv --cov=django_celery_monitor --cov-report=term --cov-report=xml --no-cov-on-fail + +basepython = + py27,flake8,flakeplus,apicheck,linkcheck,pydocstyle,cov: python2.7 + +[testenv:apicheck] +commands = + sphinx-build -W -b apicheck -d {envtmpdir}/doctrees docs docs/_build/apicheck + +[testenv:linkcheck] +commands = + sphinx-build -W -b linkcheck -d {envtmpdir}/doctrees docs docs/_build/linkcheck + +[testenv:flake8] +commands = + flake8 {toxinidir}/django_celery_monitor {toxinidir}/t + +[testenv:flakeplus] +commands = + flakeplus --2.7 {toxinidir}/django_celery_monitor {toxinidir}/t + +[testenv:pydocstyle] +commands = + pydocstyle {toxinidir}/django_celery_monitor