From dcfd45c3269605c6274b6c220fa3b120cc9c14ed Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Fri, 11 Aug 2017 16:47:37 +0100 Subject: [PATCH] Implement purge_batch on frontend cache backends --- .../contrib/wagtailfrontendcache/backends.py | 70 +++++++++++-------- wagtail/contrib/wagtailfrontendcache/tests.py | 2 +- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/wagtail/contrib/wagtailfrontendcache/backends.py b/wagtail/contrib/wagtailfrontendcache/backends.py index 22c7f5f0a..4a5d07226 100644 --- a/wagtail/contrib/wagtailfrontendcache/backends.py +++ b/wagtail/contrib/wagtailfrontendcache/backends.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, unicode_literals +from collections import defaultdict import logging import uuid @@ -23,6 +24,11 @@ class BaseBackend(object): def purge(self, url): raise NotImplementedError + def purge_batch(self, urls): + # Fallback for backends that do not support batch purging + for url in urls: + self.purge(url) + class HTTPBackend(BaseBackend): def __init__(self, params): @@ -67,7 +73,7 @@ class CloudflareBackend(BaseBackend): self.cloudflare_token = params.pop('TOKEN') self.cloudflare_zoneid = params.pop('ZONEID') - def purge(self, url): + def purge_batch(self, urls): try: purge_url = 'https://api.cloudflare.com/client/v4/zones/{0}/purge_cache'.format(self.cloudflare_zoneid) @@ -77,7 +83,7 @@ class CloudflareBackend(BaseBackend): "Content-Type": "application/json", } - data = {"files": [url]} + data = {"files": urls} response = requests.delete( purge_url, @@ -91,20 +97,20 @@ class CloudflareBackend(BaseBackend): if response.status_code != 200: response.raise_for_status() else: - logger.error("Couldn't purge '%s' from Cloudflare. Unexpected JSON parse error.", url) + logger.error("Couldn't purge from Cloudflare. Unexpected JSON parse error.") except requests.exceptions.HTTPError as e: - logger.error("Couldn't purge '%s' from Cloudflare. HTTPError: %d %s", url, e.response.status_code, e.message) - return - except requests.exceptions.InvalidURL as e: - logger.error("Couldn't purge '%s' from Cloudflare. URLError: %s", url, e.message) + logger.error("Couldn't purge from Cloudflare. HTTPError: %d %s", e.response.status_code, e.message) return if response_json['success'] is False: error_messages = ', '.join([str(err['message']) for err in response_json['errors']]) - logger.error("Couldn't purge '%s' from Cloudflare. Cloudflare errors '%s'", url, error_messages) + logger.error("Couldn't purge from Cloudflare. Cloudflare errors '%s'", error_messages) return + def purge(self, url): + self.purge_batch([url]) + class CloudfrontBackend(BaseBackend): def __init__(self, params): @@ -118,26 +124,34 @@ class CloudfrontBackend(BaseBackend): "The setting 'WAGTAILFRONTENDCACHE' requires the object 'DISTRIBUTION_ID'." ) - def purge(self, url): - url_parsed = urlparse(url) - distribution_id = None + def purge_batch(self, urls): + paths_by_distribution_id = defaultdict(list) - if isinstance(self.cloudfront_distribution_id, dict): - host = url_parsed.hostname - if host in self.cloudfront_distribution_id: - distribution_id = self.cloudfront_distribution_id.get(host) + for url in urls: + url_parsed = urlparse(url) + distribution_id = None + + if isinstance(self.cloudfront_distribution_id, dict): + host = url_parsed.hostname + if host in self.cloudfront_distribution_id: + distribution_id = self.cloudfront_distribution_id.get(host) + else: + logger.info( + "Couldn't purge '%s' from CloudFront. Hostname '%s' not found in the DISTRIBUTION_ID mapping", + url, host) else: - logger.info( - "Couldn't purge '%s' from CloudFront. Hostname '%s' not found in the DISTRIBUTION_ID mapping", - url, host) - else: - distribution_id = self.cloudfront_distribution_id + distribution_id = self.cloudfront_distribution_id - if distribution_id: - path = url_parsed.path - self._create_invalidation(distribution_id, path) + if distribution_id: + paths_by_distribution_id[distribution_id].append(url_parsed.path) - def _create_invalidation(self, distribution_id, path): + for distribution_id, paths in paths_by_distribution_id.items(): + self._create_invalidation(distribution_id, paths) + + def purge(self, url): + self.purge_batch([url]) + + def _create_invalidation(self, distribution_id, paths): import botocore try: @@ -145,15 +159,13 @@ class CloudfrontBackend(BaseBackend): DistributionId=distribution_id, InvalidationBatch={ 'Paths': { - 'Quantity': 1, - 'Items': [ - path, - ] + 'Quantity': len(paths), + 'Items': paths }, 'CallerReference': str(uuid.uuid4()) } ) except botocore.exceptions.ClientError as e: logger.error( - "Couldn't purge '%s' from CloudFront. ClientError: %s %s", path, e.response['Error']['Code'], + "Couldn't purge from CloudFront. ClientError: %s %s", e.response['Error']['Code'], e.response['Error']['Message']) diff --git a/wagtail/contrib/wagtailfrontendcache/tests.py b/wagtail/contrib/wagtailfrontendcache/tests.py index 76964b5cc..893bb136e 100644 --- a/wagtail/contrib/wagtailfrontendcache/tests.py +++ b/wagtail/contrib/wagtailfrontendcache/tests.py @@ -83,7 +83,7 @@ class TestBackendConfiguration(TestCase): backends.get('cloudfront').purge('http://www.wagtail.io/home/events/christmas/') backends.get('cloudfront').purge('http://torchbox.com/blog/') - _create_invalidation.assert_called_once_with('frontend', '/home/events/christmas/') + _create_invalidation.assert_called_once_with('frontend', ['/home/events/christmas/']) def test_multiple(self): backends = get_backends(backend_settings={