Fix pickle serialization for ImageCacheFile

When Celery CachedFileBackend used with filesystem storage (django.core.files.storage.FileSystemStorage), everything works fine.
But there are issues with storages.backends.s3boto3.S3Boto3Storage (and it's fix from #391), as well as with django_s3_storage.storage.S3Storage.

Exception was:

```
Traceback (most recent call last):
  ...

  File "/src/django-imagekit/imagekit/cachefiles/__init__.py", line 131, in __bool__
    existence_required.send(sender=self, file=self)
  ...
  File "/libs/utils.py", line 380, in on_existence_required
    file.generate()
  File "/src/django-imagekit/imagekit/cachefiles/__init__.py", line 94, in generate
    self.cachefile_backend.generate(self, force)
  File "/src/django-imagekit/imagekit/cachefiles/backends.py", line 136, in generate
    self.schedule_generation(file, force=force)
  File "/src/django-imagekit/imagekit/cachefiles/backends.py", line 165, in schedule_generation
    _celery_task.delay(self, file.generator, force=force)
  ...
  File "/lib/python3.6/site-packages/kombu/serialization.py", line 221, in dumps
    payload = encoder(data)
  File "/lib/python3.6/site-packages/kombu/serialization.py", line 350, in pickle_dumps
    return dumper(obj, protocol=pickle_protocol)
kombu.exceptions.EncodeError: can't pickle _thread._local objects
```
This commit is contained in:
Roman Gorbil 2017-10-20 18:26:37 +07:00 committed by Venelin Stoykov
parent 7e23384145
commit de991d4048
3 changed files with 25 additions and 1 deletions

View file

@ -185,6 +185,11 @@ Or, in Python:
def on_source_saved(self, file):
file.generate()
.. note::
If you use custom storage backend for some specs,
(storage passed to the field different than configured one)
it's required the storage to be pickleable
__ https://pypi.python.org/pypi/django-celery

View file

@ -144,8 +144,24 @@ class ImageCacheFile(BaseIKFile, ImageFile):
# file is hidden link to "file" attribute
state.pop('_file', None)
# remove storage from state as some non-FileSystemStorage can't be
# pickled
settings_storage = get_singleton(
settings.IMAGEKIT_DEFAULT_FILE_STORAGE,
'file storage backend'
)
if state['storage'] == settings_storage:
state.pop('storage')
return state
def __setstate__(self, state):
if 'storage' not in state:
state['storage'] = get_singleton(
settings.IMAGEKIT_DEFAULT_FILE_STORAGE,
'file storage backend'
)
self.__dict__.update(state)
def __nonzero__(self):
# Python 2 compatibility
return self.__bool__()

View file

@ -37,4 +37,7 @@ def test_cachefiles():
# remove link to file from spec source generator
# test __getstate__ of ImageCacheFile
file.generator.source = None
pickleback(file)
restored_file = pickleback(file)
assert file is not restored_file
# Assertion for #437 and #451
assert file.storage is restored_file.storage