Fix build, remove unmaintained Django & Python versions.

Update .travis.yml

Update tox.ini

Update test_cachefiles.py

Update test_optimistic_strategy.py

Update test_no_extra_queries.py

Update test_cachefiles.py

Update .travis.yml

Update tox.ini

Update .travis.yml

.travis.yml

@@ -1,62 +1,35 @@
+sudo: false
 
 language: python
-
+python:
-sudo: false
+  - "3.8"
+  - "3.7"
+  - "3.6"
+  - "3.5"
+env:
+  - DJANGO="master"
+  - DJANGO="30"
+  - DJANGO="22"
+  - DJANGO="21"
+  - DJANGO="21"
+  - DJANGO="20"
+  - DJANGO="111"
 
 install:
   - pip install tox
 
 script:
-  - tox
+  - tox -e py$(python -c 'import sys;print("".join(map(str, sys.version_info[:2])))')-django${DJANGO}
 
-matrix:
+jobs:
   fast_finish: true
-  include:
+  allow_failures:
-    - env: TOXENV=py27-django14
+    - env: DJANGO="master"
-      python: 2.7
+  exclude:
-    - env: TOXENV=py27-django15
+    - python: "3.5"
-      python: 2.7
+      env: DJANGO="30"
-    - env: TOXENV=py27-django16
+    - python: "3.5"
-      python: 2.7
+      env: DJANGO="master"
-    - env: TOXENV=py27-django17
-      python: 2.7
-    - env: TOXENV=py27-django18
-      python: 2.7
-    - env: TOXENV=py27-django19
-      python: 2.7
-    - env: TOXENV=py27-django110
-      python: 2.7
-    - env: TOXENV=py27-django111
-      python: 2.7
-    - env: TOXENV=py33-django15
-      python: 3.3
-    - env: TOXENV=py33-django16
-      python: 3.3
-    - env: TOXENV=py33-django17
-      python: 3.3
-    - env: TOXENV=py33-django18
-      python: 3.3
-    - env: TOXENV=py34-django16
-      python: 3.4
-    - env: TOXENV=py34-django17
-      python: 3.4
-    - env: TOXENV=py34-django18
-      python: 3.4
-    - env: TOXENV=py34-django19
-      python: 3.4
-    - env: TOXENV=py34-django110
-      python: 3.4
-    - env: TOXENV=py34-django111
-      python: 3.4
-    - env: TOXENV=py35-django18
-      python: 3.5
-    - env: TOXENV=py35-django19
-      python: 3.5
-    - env: TOXENV=py35-django110
-      python: 3.5
-    - env: TOXENV=py35-django111
-      python: 3.5
-
 
 notifications:
   irc: "irc.freenode.org#imagekit"

README.rst

@@ -39,6 +39,7 @@ Installation
 Usage Overview
 ==============
 
+.. _specs:
 
 Specs
 -----
@@ -70,8 +71,8 @@ your model class:
                                           options={'quality': 60})
 
     profile = Profile.objects.all()[0]
-    print profile.avatar_thumbnail.url    # > /media/CACHE/images/982d5af84cddddfd0fbf70892b4431e4.jpg
+    print(profile.avatar_thumbnail.url)    # > /media/CACHE/images/982d5af84cddddfd0fbf70892b4431e4.jpg
-    print profile.avatar_thumbnail.width  # > 100
+    print(profile.avatar_thumbnail.width)  # > 100
 
 As you can probably tell, ImageSpecFields work a lot like Django's
 ImageFields. The difference is that they're automatically generated by
@@ -97,8 +98,8 @@ class:
                                                options={'quality': 60})
 
     profile = Profile.objects.all()[0]
-    print profile.avatar_thumbnail.url    # > /media/avatars/MY-avatar.jpg
+    print(profile.avatar_thumbnail.url)    # > /media/avatars/MY-avatar.jpg
-    print profile.avatar_thumbnail.width  # > 100
+    print(profile.avatar_thumbnail.width)  # > 100
 
 This is pretty similar to our previous example. We don't need to specify a
 "source" any more since we're not processing another image field, but we do need
@@ -144,7 +145,7 @@ on, or what should be done with the result; that's up to you:
 
 .. code-block:: python
 
-    source_file = open('/path/to/myimage.jpg')
+    source_file = open('/path/to/myimage.jpg', 'rb')
     image_generator = Thumbnail(source=source_file)
     result = image_generator.generate()
 
@@ -159,7 +160,7 @@ example, if you wanted to save it to disk:
 
 .. code-block:: python
 
-    dest = open('/path/to/dest.jpg', 'w')
+    dest = open('/path/to/dest.jpg', 'wb')
     dest.write(result.read())
     dest.close()
 

docs/advanced_usage.rst

@@ -163,7 +163,7 @@ A simple example of a custom source group class is as follows:
         def files(self):
             os.chdir(self.dir)
             for name in glob.glob('*.jpg'):
-                yield open(name)
+                yield open(name, 'rb')
 
 Instances of this class could then be registered with one or more spec id:
 

docs/caching.rst

@@ -3,7 +3,7 @@ Caching
 
 
 Default Backend Workflow
-================
+========================
 
 
 ``ImageSpec``
@@ -29,6 +29,8 @@ objects, but they've got a little trick up their sleeve: they represent files
 that may not actually exist!
 
 
+.. _cache-file-strategy:
+
 Cache File Strategy
 -------------------
 
@@ -55,6 +57,8 @@ The default strategy only defines the first two of these, as follows:
             file.generate()
 
 
+.. _cache-file-backend:
+
 Cache File Backend
 ------------------
 
@@ -185,6 +189,11 @@ Or, in Python:
             def on_source_saved(self, file):
                 file.generate()
 
+.. note::
+
+    If you use custom storage backend for some specs,
+    (storage passed to the field different than configured one)
+    it's required the storage to be pickleable
 
 
 __ https://pypi.python.org/pypi/django-celery

docs/conf.py

@@ -54,7 +54,7 @@ execfile(os.path.join(os.path.dirname(__file__), '..', 'imagekit',
 # built documents.
 #
 # The short X.Y version.
-version = re.match('\d+\.\d+', pkgmeta['__version__']).group()
+version = re.match(r'\d+\.\d+', pkgmeta['__version__']).group()
 # The full version, including alpha/beta/rc tags.
 release = pkgmeta['__version__']
 

docs/upgrading.rst

@@ -79,12 +79,9 @@ IK3 provides analogous settings for cache file backends and strategies:
     IMAGEKIT_DEFAULT_CACHEFILE_BACKEND = 'path.to.MyCacheFileBackend'
     IMAGEKIT_DEFAULT_CACHEFILE_STRATEGY = 'path.to.MyCacheFileStrategy'
 
-See the documentation on `cache file backends`_ and `cache file strategies`_
+See the documentation on :ref:`cache file backends <cache-file-backend>` and :ref:`cache file strategies <cache-file-strategy>`
 for more details.
 
-.. _`cache file backends`:
-.. _`cache file strategies`:
-
 
 Conditional model ``processors``
 --------------------------------
@@ -93,9 +90,7 @@ In IK2, an ``ImageSpecField`` could take a ``processors`` callable instead of
 an iterable, which allowed processing decisions to made based on other
 properties of the model. IK3 does away with this feature for consistency's sake
 (if one kwarg could be callable, why not all?), but provides a much more robust
-solution: the custom ``spec``. See the `advanced usage`_ documentation for more.
+solution: the custom ``spec``. See the :doc:`advanced usage <advanced_usage>` documentation for more.
-
-.. _`advanced usage`:
 
 
 Conditonal ``cache_to`` file names
@@ -109,9 +104,7 @@ There is a way to achieve custom file names by overriding your spec's
 ``cachefile_name``, but it is not recommended, as the spec's default
 behavior is to hash the combination of ``source``, ``processors``, ``format``,
 and other spec options to ensure that changes to the spec always result in
-unique file names. See the documentation on `specs`_ for more.
+unique file names. See the documentation on :ref:`specs` for more.
-
-.. _`specs`:
 
 
 Processors have moved to PILKit

imagekit/cachefiles/__init__.py

@@ -3,6 +3,7 @@ from django.conf import settings
 from django.core.files import File
 from django.core.files.images import ImageFile
 from django.utils.functional import SimpleLazyObject
+from django.utils.encoding import smart_str
 from ..files import BaseIKFile
 from ..registry import generator_registry
 from ..signals import content_required, existence_required
@@ -143,12 +144,33 @@ class ImageCacheFile(BaseIKFile, ImageFile):
         # file is hidden link to "file" attribute
         state.pop('_file', None)
 
+        # remove storage from state as some non-FileSystemStorage can't be
+        # pickled
+        settings_storage = get_singleton(
+            settings.IMAGEKIT_DEFAULT_FILE_STORAGE,
+            'file storage backend'
+        )
+        if state['storage'] == settings_storage:
+            state.pop('storage')
         return state
 
+    def __setstate__(self, state):
+        if 'storage' not in state:
+            state['storage'] = get_singleton(
+                settings.IMAGEKIT_DEFAULT_FILE_STORAGE,
+                'file storage backend'
+            )
+        self.__dict__.update(state)
+
     def __nonzero__(self):
         # Python 2 compatibility
         return self.__bool__()
 
+    def __repr__(self):
+        return smart_str("<%s: %s>" % (
+            self.__class__.__name__, self if self.name else "None")
+        )
+
 
 class LazyImageCacheFile(SimpleLazyObject):
     def __init__(self, generator_id, *args, **kwargs):

imagekit/cachefiles/backends.py

@@ -110,7 +110,7 @@ class Simple(CachedFileBackend):
 
     def _exists(self, file):
         return bool(getattr(file, '_file', None)
-                    or file.storage.exists(file.name))
+                    or (file.name and file.storage.exists(file.name)))
 
 
 def _generate_file(backend, file, force=False):

imagekit/files.py

@@ -49,7 +49,7 @@ class BaseIKFile(File):
 
     def _get_size(self):
         self._require_file()
-        if not self._committed:
+        if not getattr(self, '_committed', False):
             return self.file.size
         return self.storage.size(self.name)
     size = property(_get_size)

imagekit/management/commands/generateimages.py

@@ -14,11 +14,15 @@ match both. Subsegments are always matched, so "a" will match "a" as
 well as "a:b" and "a:b:c".""")
     args = '[generator_ids]'
 
+    def add_arguments(self, parser):
+        parser.add_argument('generator_id', nargs='*', help='<app_name>:<model>:<field> for model specs')
+
     def handle(self, *args, **options):
         generators = generator_registry.get_ids()
 
-        if args:
+        generator_ids = options['generator_id'] if 'generator_id' in options else args
-            patterns = self.compile_patterns(args)
+        if generator_ids:
+            patterns = self.compile_patterns(generator_ids)
             generators = (id for id in generators if any(p.match(id) for p in patterns))
 
         for generator_id in generators:

imagekit/models/fields/__init__.py

@@ -93,7 +93,7 @@ class ProcessedImageField(models.ImageField, SpecHostField):
 
     def __init__(self, processors=None, format=None, options=None,
             verbose_name=None, name=None, width_field=None, height_field=None,
-            autoconvert=True, spec=None, spec_id=None, **kwargs):
+            autoconvert=None, spec=None, spec_id=None, **kwargs):
         """
         The ProcessedImageField constructor accepts all of the arguments that
         the :class:`django.db.models.ImageField` constructor accepts, as well
@@ -101,6 +101,10 @@ class ProcessedImageField(models.ImageField, SpecHostField):
         :class:`imagekit.models.ImageSpecField`.
 
         """
+        # if spec is not provided then autoconvert will be True by default
+        if spec is None and autoconvert is None:
+            autoconvert = True
+
         SpecHost.__init__(self, processors=processors, format=format,
                 options=options, autoconvert=autoconvert, spec=spec,
                 spec_id=spec_id)

imagekit/pkgmeta.py

@@ -1,5 +1,5 @@
 __title__ = 'django-imagekit'
 __author__ = 'Matthew Tretter, Venelin Stoykov, Eric Eldredge, Bryan Veloso, Greg Newman, Chris Drackett, Justin Driscoll'
-__version__ = '4.0.1'
+__version__ = '4.0.2'
 __license__ = 'BSD'
 __all__ = ['__title__', '__author__', '__version__', '__license__']

imagekit/specs/__init__.py

@@ -143,22 +143,25 @@ class ImageSpec(BaseImageSpec):
             raise MissingSource("The spec '%s' has no source file associated"
                                 " with it." % self)
 
-        file_opened_locally = False
         # TODO: Move into a generator base class
         # TODO: Factor out a generate_image function so you can create a generator and only override the PIL.Image creating part. (The tricky part is how to deal with original_format since generator base class won't have one.)
+
+        closed = self.source.closed
+        if closed:
+            # Django file object should know how to reopen itself if it was closed
+            # https://code.djangoproject.com/ticket/13750
+            self.source.open()
+
         try:
             img = open_image(self.source)
-        except ValueError:
+            new_image = process_image(img,
-
+                                      processors=self.processors,
-            # Re-open the file -- https://code.djangoproject.com/ticket/13750
+                                      format=self.format,
-            self.source.open()
+                                      autoconvert=self.autoconvert,
-            file_opened_locally = True
-            img = open_image(self.source)
-
-        new_image =  process_image(img, processors=self.processors,
-                                   format=self.format, autoconvert=self.autoconvert,
                                       options=self.options)
-        if file_opened_locally:
+        finally:
+            if closed:
+                # We need to close the file if it was opened by us
                 self.source.close()
         return new_image
 

tests/models.py

@@ -1,10 +1,17 @@
 from django.db import models
 
+from imagekit import ImageSpec
 from imagekit.models import ProcessedImageField
 from imagekit.models import ImageSpecField
 from imagekit.processors import Adjust, ResizeToFill, SmartCrop
 
 
+class Thumbnail(ImageSpec):
+    processors = [ResizeToFill(100, 60)]
+    format = 'JPEG'
+    options = {'quality': 60}
+
+
 class ImageModel(models.Model):
     image = models.ImageField(upload_to='b')
 
@@ -27,6 +34,10 @@ class ProcessedImageFieldModel(models.Model):
             options={'quality': 90}, upload_to='p')
 
 
+class ProcessedImageFieldWithSpecModel(models.Model):
+    processed = ProcessedImageField(spec=Thumbnail, upload_to='p')
+
+
 class CountingCacheFileStrategy(object):
     def __init__(self):
         self.on_existence_required_count = 0

tests/test_cachefiles.py

@@ -1,3 +1,4 @@
+from unittest import mock
 from django.conf import settings
 from hashlib import md5
 from imagekit.cachefiles import ImageCacheFile, LazyImageCacheFile
@@ -48,6 +49,31 @@ def test_no_source_error():
     file.generate()
 
 
+def test_repr_does_not_send_existence_required():
+    """
+    Ensure that `__repr__` method does not send `existance_required` signal
+
+    Cachefile strategy may be configured to generate file on
+    `existance_required`.
+    To generate images, backend passes `ImageCacheFile` instance to worker.
+    Both celery and RQ calls `__repr__` method for each argument to enque call.
+    And if `__repr__` of object will send this signal, we will get endless
+    recursion
+
+    """
+    with mock.patch('imagekit.cachefiles.existence_required') as signal:
+        # import here to apply mock
+        from imagekit.cachefiles import ImageCacheFile
+
+        spec = TestSpec(source=get_unique_image_file())
+        file = ImageCacheFile(
+            spec,
+            cachefile_backend=DummyAsyncCacheFileBackend()
+        )
+        file.__repr__()
+        eq_(signal.send.called, False)
+
+
 def test_memcached_cache_key():
     """
     Ensure the default cachefile backend is sanitizing its cache key for

tests/test_closing_fieldfiles.py

@@ -0,0 +1,25 @@
+from nose.tools import assert_false, assert_true
+
+from .models import Thumbnail
+from .utils import create_photo
+
+
+def test_do_not_leak_open_files():
+    instance = create_photo('leak-test.jpg')
+    source_file = instance.original_image
+    # Ensure the FieldFile is closed before generation
+    source_file.close()
+    image_generator = Thumbnail(source=source_file)
+    image_generator.generate()
+    assert_true(source_file.closed)
+
+
+def test_do_not_close_open_files_after_generate():
+    instance = create_photo('do-not-close-test.jpg')
+    source_file = instance.original_image
+    # Ensure the FieldFile is opened before generation
+    source_file.open()
+    image_generator = Thumbnail(source=source_file)
+    image_generator.generate()
+    assert_false(source_file.closed)
+    source_file.close()

tests/test_fields.py

@@ -5,7 +5,9 @@ from imagekit import forms as ikforms
 from imagekit.processors import SmartCrop
 from nose.tools import eq_
 from . import imagegenerators  # noqa
-from .models import ProcessedImageFieldModel, ImageModel
+from .models import (ProcessedImageFieldModel,
+                     ProcessedImageFieldWithSpecModel,
+                     ImageModel)
 from .utils import get_image_file
 
 
@@ -19,6 +21,16 @@ def test_model_processedimagefield():
     eq_(instance.processed.height, 50)
 
 
+def test_model_processedimagefield_with_spec():
+    instance = ProcessedImageFieldWithSpecModel()
+    file = File(get_image_file())
+    instance.processed.save('whatever.jpeg', file)
+    instance.save()
+
+    eq_(instance.processed.width, 100)
+    eq_(instance.processed.height, 60)
+
+
 def test_form_processedimagefield():
     class TestForm(forms.ModelForm):
         image = ikforms.ProcessedImageField(spec_id='tests:testform_image',

tests/test_generateimage_tag.py

@@ -30,7 +30,7 @@ def test_dangling_html_attrs_delimiter():
 @raises(TemplateSyntaxError)
 def test_html_attrs_assignment():
     """
-    You can either use generateimage as an assigment tag or specify html attrs,
+    You can either use generateimage as an assignment tag or specify html attrs,
     but not both.
 
     """

tests/test_no_extra_queries.py

@@ -1,5 +1,5 @@
 from nose.tools import assert_false
-from mock import Mock, PropertyMock, patch
+from unittest.mock import Mock, PropertyMock, patch
 from .models import Photo
 
 

tests/test_optimistic_strategy.py

@@ -1,6 +1,6 @@
 from nose.tools import assert_true, assert_false
 from imagekit.cachefiles import ImageCacheFile
-from mock import Mock
+from unittest.mock import Mock
 from .utils import create_image
 from django.core.files.storage import FileSystemStorage
 from imagekit.cachefiles.backends import Simple as SimpleCFBackend

tests/test_serialization.py

@@ -37,4 +37,7 @@ def test_cachefiles():
     # remove link to file from spec source generator
     # test __getstate__ of ImageCacheFile
     file.generator.source = None
-    pickleback(file)
+    restored_file = pickleback(file)
+    assert file is not restored_file
+    # Assertion for #437 and #451
+    assert file.storage is restored_file.storage

tests/test_thumbnail_tag.py

@@ -42,7 +42,7 @@ def test_too_many_args():
 @raises(TemplateSyntaxError)
 def test_html_attrs_assignment():
     """
-    You can either use thumbnail as an assigment tag or specify html attrs,
+    You can either use thumbnail as an assignment tag or specify html attrs,
     but not both.
 
     """

tests/utils.py

@@ -64,7 +64,7 @@ def render_tag(ttag):
 
 
 def get_html_attrs(ttag):
-    return BeautifulSoup(render_tag(ttag)).img.attrs
+    return BeautifulSoup(render_tag(ttag), features="html.parser").img.attrs
 
 
 def assert_file_is_falsy(file):

tox.ini

@@ -1,141 +1,18 @@
 [tox]
 envlist =
-    py35-django111, py35-django110, py35-django19, py35-django18,
+    py38-django{master,30,22,21,20,111},
-    py34-django111, py34-django110, py34-django19, py34-django18, py34-django17, py34-django16,
+    py37-django{master,30,22,21,20,111},
-    py33-django18, py33-django17, py33-django16, py33-django15,
+    py36-django{master,30,22,21,20,111},
-    py27-django111, py27-django110, py27-django19, py27-django18, py27-django17, py27-django16, py27-django15, py27-django14,
+    py35-django{21,20,111},
 
 [testenv]
 commands = python setup.py test
 
-[testenv:py35-django111]
-basepython = python3.5
 deps =
-    Django>=1.11a1,<1.12
+    djangomaster: git+https://github.com/django/django.git@master#egg=Django
-    django-nose==1.4.4
+    django30: Django>=3.0,<3.1
-
+    django22: Django>=2.2,<3.0
-[testenv:py35-django110]
+    django21: Django>=2.1,<2.2
-basepython = python3.5
+    django20: Django>=2.0,<2.1
-deps =
+    django111: Django>=1.11,<2.0
-    Django>=1.10,<1.11
+    django{21,20,111}: django-nose==1.4.5
-    django-nose==1.4.4
-
-[testenv:py35-django19]
-basepython = python3.5
-deps =
-    Django>=1.9,<1.10
-    django-nose==1.4.2
-
-[testenv:py35-django18]
-basepython = python3.5
-deps =
-    Django>=1.8,<1.9
-    django-nose==1.4.2
-
-[testenv:py34-django111]
-basepython = python3.4
-deps =
-    Django>=1.11a1,<1.12
-    django-nose==1.4.4
-
-[testenv:py34-django110]
-basepython = python3.4
-deps =
-    Django>=1.10,<1.11
-    django-nose==1.4.4
-
-[testenv:py34-django19]
-basepython = python3.4
-deps =
-    Django>=1.9,<1.10
-    django-nose==1.4.2
-
-[testenv:py34-django18]
-basepython = python3.4
-deps =
-    Django>=1.8,<1.9
-    django-nose==1.4.2
-
-[testenv:py34-django17]
-basepython = python3.4
-deps =
-    Django>=1.7,<1.8
-    django-nose==1.4
-
-[testenv:py34-django16]
-basepython = python3.4
-deps =
-    Django>=1.6,<1.7
-    django-nose<=1.4.2
-
-[testenv:py33-django18]
-basepython = python3.3
-deps =
-    Django>=1.8,<1.9
-    django-nose==1.4.2
-
-[testenv:py33-django17]
-basepython = python3.3
-deps =
-    Django>=1.7,<1.8
-    django-nose==1.4
-
-[testenv:py33-django16]
-basepython = python3.3
-deps =
-    Django>=1.6,<1.7
-    django-nose<=1.4.2
-
-[testenv:py33-django15]
-basepython = python3.3
-deps =
-    Django>=1.5,<1.6
-    django-nose==1.4
-
-[testenv:py27-django111]
-basepython = python2.7
-deps =
-    Django>=1.11a1,<1.12
-    django-nose==1.4.4
-
-[testenv:py27-django110]
-basepython = python2.7
-deps =
-    Django>=1.10,<1.11
-    django-nose==1.4.4
-
-[testenv:py27-django19]
-basepython = python2.7
-deps =
-    Django>=1.9,<1.10
-    django-nose==1.4.2
-
-[testenv:py27-django18]
-basepython = python2.7
-deps =
-    Django>=1.8,<1.9
-    django-nose==1.4.2
-
-[testenv:py27-django17]
-basepython = python2.7
-deps =
-    Django>=1.7,<1.8
-    django-nose==1.4
-
-[testenv:py27-django16]
-basepython = python2.7
-deps =
-    Django>=1.6,<1.7
-    django-nose<=1.4.2
-
-[testenv:py27-django15]
-basepython = python2.7
-deps =
-    Django>=1.5,<1.6
-    django-nose==1.4
-
-[testenv:py27-django14]
-basepython = python2.7
-deps =
-    Django>=1.4,<1.5
-    django-nose==1.4