From 8c8b04774087494a3980488132c926332470cc59 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 8 Jul 2015 17:08:22 +0100 Subject: [PATCH 1/4] Rewrote frontendcache HTTPBackend to use urllib --- .../contrib/wagtailfrontendcache/backends.py | 92 +++++++++---------- 1 file changed, 43 insertions(+), 49 deletions(-) diff --git a/wagtail/contrib/wagtailfrontendcache/backends.py b/wagtail/contrib/wagtailfrontendcache/backends.py index 224e00dd2..9496e2fc8 100644 --- a/wagtail/contrib/wagtailfrontendcache/backends.py +++ b/wagtail/contrib/wagtailfrontendcache/backends.py @@ -1,9 +1,9 @@ import logging -import requests +import json -from django.utils.six.moves.urllib.parse import urlparse - -from requests.adapters import HTTPAdapter +from django.utils.six.moves.urllib.parse import urlparse, urlunparse, urlencode +from django.utils.six.moves.urllib.request import Request, urlopen +from django.utils.six.moves.urllib.error import URLError, HTTPError logger = logging.getLogger('wagtail.frontendcache') @@ -15,44 +15,40 @@ class BaseBackend(object): class HTTPBackend(BaseBackend): - - class CustomHTTPAdapter(HTTPAdapter): - """ - Requests will always send requests to whatever server is in the netloc - part of the URL. This is a problem with purging the cache as this netloc - may point to a different server (such as an nginx instance running in - front of the cache). - - This class allows us to send a purge request directly to the cache server - with the host header still set correctly. It does this by changing the "url" - parameter of get_connection to always point to the cache server. Requests - will then use this connection to purge the page. - """ - def __init__(self, cache_url): - self.cache_url = cache_url - super(HTTPBackend.CustomHTTPAdapter, self).__init__() - - def get_connection(self, url, proxies=None): - return super(HTTPBackend.CustomHTTPAdapter, self).get_connection(self.cache_url, proxies) - - def __init__(self, params): - self.cache_location = params.pop('LOCATION') - - self.session = requests.Session() - self.session.mount('http://', self.CustomHTTPAdapter(self.cache_location)) + location_url_parsed = urlparse(params.pop('LOCATION')) + self.cache_scheme = location_url_parsed.scheme + self.cache_netloc = location_url_parsed.netloc def purge(self, url): - try: - response = self.session.request('PURGE', url) - except requests.ConnectionError: - logger.error("Couldn't purge '%s' from HTTP cache: Connection error", url) - return + url_parsed = urlparse(url) + host = url_parsed.hostname - # Check for error - if response.status_code != 200: - logger.error("Couldn't purge '%s' from HTTP cache: Didn't recieve a 200 response (instead, we got '%d %s')", url, response.status_code, response.reason) - return + # Append port to host if it is set in the original URL + if url_parsed.port: + host += (':' + str(url_parsed.port)) + + request = Request( + url=urlunparse([ + self.cache_scheme, + self.cache_netloc, + url_parsed.path, + url_parsed.params, + url_parsed.query, + url_parsed.fragment + ]), + headers={ + 'Host': host, + }, + method='PURGE' + ) + + try: + urlopen(request) + except HTTPError as e: + logger.error("Couldn't purge '%s' from HTTP cache. HTTPError: %d %s", url, e.code, e.reason) + except URLError as e: + logger.error("Couldn't purge '%s' from HTTP cache. URLError: %s", url, e.reason) class CloudflareBackend(BaseBackend): @@ -62,23 +58,21 @@ class CloudflareBackend(BaseBackend): def purge(self, url): try: - response = requests.post('https://www.cloudflare.com/api_json.html', { + response = urlopen('https://www.cloudflare.com/api_json.html', data=urlencode({ 'email': self.cloudflare_email, 'tkn': self.cloudflare_token, 'a': 'zone_file_purge', 'z': urlparse(url).netloc, 'url': url - }) - except requests.ConnectionError: - logger.error("Couldn't purge '%s' from Cloudflare: Connection error", url) + }).encode('utf-8')) + except HTTPError as e: + logger.error("Couldn't purge '%s' from Cloudflare. HTTPError: %d %s", url, e.code, e.reason) + return + except URLError as e: + logger.error("Couldn't purge '%s' from Cloudflare. URLError: %s", url, e.reason) return - # Check for error - if response.status_code != 200: - logger.error("Couldn't purge '%s' from Cloudflare: Didn't recieve a 200 response (instead, we got '%d %s')", url, response.status_code, response.reason) - return - - response_json = response.json() + response_json = json.loads(response.read().decode('utf-8')) if response_json['result'] == 'error': - logger.error("Couldn't purge '%s' from Cloudflare: Cloudflare error '%s'", url, response_json['msg']) + logger.error("Couldn't purge '%s' from Cloudflare. Cloudflare error '%s'", url, response_json['msg']) return From 0731e77c0f6f43ba838dc384990133b15dc3dea9 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 8 Jul 2015 17:25:38 +0100 Subject: [PATCH 2/4] Removed dependency on requests --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 73849d488..4749664ff 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ install_requires = [ "beautifulsoup4>=4.3.2", "html5lib==0.999", "Unidecode>=0.04.14", - 'requests>=2.0.0', "Willow==0.2.1", ] From 064aaa654bd2aac648c90ede0ad2101b9ddd6ba7 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 8 Jul 2015 17:35:41 +0100 Subject: [PATCH 3/4] Set the User-Agent header when purging cache --- wagtail/contrib/wagtailfrontendcache/backends.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wagtail/contrib/wagtailfrontendcache/backends.py b/wagtail/contrib/wagtailfrontendcache/backends.py index 9496e2fc8..f924e1cd1 100644 --- a/wagtail/contrib/wagtailfrontendcache/backends.py +++ b/wagtail/contrib/wagtailfrontendcache/backends.py @@ -5,6 +5,8 @@ from django.utils.six.moves.urllib.parse import urlparse, urlunparse, urlencode from django.utils.six.moves.urllib.request import Request, urlopen from django.utils.six.moves.urllib.error import URLError, HTTPError +from wagtail.wagtailcore import __version__ + logger = logging.getLogger('wagtail.frontendcache') @@ -39,6 +41,7 @@ class HTTPBackend(BaseBackend): ]), headers={ 'Host': host, + 'User-Agent': 'Wagtail-frontendcache/' + __version__ }, method='PURGE' ) From 4b49c0ea9e6a450987c8440712e96584c7746f3d Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Thu, 9 Jul 2015 10:20:13 +0100 Subject: [PATCH 4/4] Update tests --- wagtail/contrib/wagtailfrontendcache/tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wagtail/contrib/wagtailfrontendcache/tests.py b/wagtail/contrib/wagtailfrontendcache/tests.py index 52751c740..69825ffe3 100644 --- a/wagtail/contrib/wagtailfrontendcache/tests.py +++ b/wagtail/contrib/wagtailfrontendcache/tests.py @@ -25,7 +25,8 @@ class TestBackendConfiguration(TestCase): self.assertEqual(set(backends.keys()), set(['varnish'])) self.assertIsInstance(backends['varnish'], HTTPBackend) - self.assertEqual(backends['varnish'].cache_location, 'http://localhost:8000') + self.assertEqual(backends['varnish'].cache_scheme, 'http') + self.assertEqual(backends['varnish'].cache_netloc, 'localhost:8000') def test_cloudflare(self): backends = get_backends(backend_settings={ @@ -78,7 +79,8 @@ class TestBackendConfiguration(TestCase): self.assertEqual(set(backends.keys()), set(['default'])) self.assertIsInstance(backends['default'], HTTPBackend) - self.assertEqual(backends['default'].cache_location, 'http://localhost:8000') + self.assertEqual(backends['default'].cache_scheme, 'http') + self.assertEqual(backends['default'].cache_netloc, 'localhost:8000') PURGED_URLS = []