2014-02-28 23:12:34 +00:00
|
|
|
# Copyright (C) 2004-2014 Bastian Kleineidam
|
2005-12-07 21:55:16 +00:00
|
|
|
#
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
|
# it under the terms of the GNU General Public License as published by
|
|
|
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
|
|
|
# (at your option) any later version.
|
|
|
|
|
#
|
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
|
#
|
2009-07-24 21:58:20 +00:00
|
|
|
# You should have received a copy of the GNU General Public License along
|
|
|
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
|
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
Define http test support classes for LinkChecker tests.
|
|
|
|
|
"""
|
|
|
|
|
|
2020-05-14 19:15:28 +00:00
|
|
|
import html
|
2020-04-19 18:05:55 +00:00
|
|
|
from http.server import CGIHTTPRequestHandler, SimpleHTTPRequestHandler, HTTPServer
|
2019-11-11 20:12:25 +00:00
|
|
|
from http.client import HTTPConnection, HTTPSConnection
|
2020-04-19 18:05:55 +00:00
|
|
|
import os.path
|
2019-11-11 20:12:25 +00:00
|
|
|
import ssl
|
2005-12-07 21:55:16 +00:00
|
|
|
import time
|
2011-05-28 17:24:38 +00:00
|
|
|
import threading
|
2020-05-14 19:15:28 +00:00
|
|
|
import urllib.parse
|
2019-09-15 18:49:33 +00:00
|
|
|
from io import BytesIO
|
2008-05-20 17:01:16 +00:00
|
|
|
from . import LinkCheckTest
|
2019-11-11 20:12:25 +00:00
|
|
|
from .. import get_file
|
2005-12-07 21:55:16 +00:00
|
|
|
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
class StoppableHttpRequestHandler(SimpleHTTPRequestHandler):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
HTTP request handler with QUIT stopping the server.
|
|
|
|
|
"""
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def do_QUIT(self):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
Send 200 OK response, and set server.stop to True.
|
|
|
|
|
"""
|
|
|
|
|
self.send_response(200)
|
|
|
|
|
self.end_headers()
|
|
|
|
|
self.server.stop = True
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def log_message(self, format, *args):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
Logging is disabled.
|
|
|
|
|
"""
|
|
|
|
|
pass
|
|
|
|
|
|
2020-05-28 19:29:13 +00:00
|
|
|
|
2008-06-16 19:51:58 +00:00
|
|
|
# serve .xhtml files as application/xhtml+xml
|
2020-05-28 19:29:13 +00:00
|
|
|
StoppableHttpRequestHandler.extensions_map.update(
|
|
|
|
|
{".xhtml": "application/xhtml+xml",}
|
|
|
|
|
)
|
2008-06-16 19:51:58 +00:00
|
|
|
|
2005-12-07 21:55:16 +00:00
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
class StoppableHttpServer(HTTPServer):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
HTTP server that reacts to self.stop flag.
|
|
|
|
|
"""
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def serve_forever(self):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
Handle one request at a time until stopped.
|
|
|
|
|
"""
|
|
|
|
|
self.stop = False
|
|
|
|
|
while not self.stop:
|
|
|
|
|
self.handle_request()
|
|
|
|
|
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
class NoQueryHttpRequestHandler(StoppableHttpRequestHandler):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
2012-10-15 12:36:27 +00:00
|
|
|
Handler ignoring the query part of requests and sending dummy directory
|
|
|
|
|
listings.
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def remove_path_query(self):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
Remove everything after a question mark.
|
|
|
|
|
"""
|
2020-05-28 19:29:13 +00:00
|
|
|
i = self.path.find("?")
|
2005-12-07 21:55:16 +00:00
|
|
|
if i != -1:
|
|
|
|
|
self.path = self.path[:i]
|
|
|
|
|
|
2012-11-13 17:11:25 +00:00
|
|
|
def get_status(self):
|
2020-05-28 19:29:13 +00:00
|
|
|
dummy, status = self.path.rsplit("/", 1)
|
2012-11-13 17:11:25 +00:00
|
|
|
status = int(status)
|
|
|
|
|
if status in self.responses:
|
2020-05-28 19:29:13 +00:00
|
|
|
return status
|
2012-11-13 17:11:25 +00:00
|
|
|
return 500
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def do_GET(self):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
Removes query part of GET request.
|
|
|
|
|
"""
|
|
|
|
|
self.remove_path_query()
|
2012-11-13 17:11:25 +00:00
|
|
|
if "status/" in self.path:
|
|
|
|
|
status = self.get_status()
|
|
|
|
|
self.send_response(status)
|
|
|
|
|
self.end_headers()
|
2020-05-28 19:29:13 +00:00
|
|
|
if status >= 200 and status not in (204, 304):
|
2019-09-15 18:49:33 +00:00
|
|
|
self.wfile.write(b"testcontent")
|
2012-11-13 17:11:25 +00:00
|
|
|
else:
|
|
|
|
|
super(NoQueryHttpRequestHandler, self).do_GET()
|
2005-12-07 21:55:16 +00:00
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def do_HEAD(self):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
Removes query part of HEAD request.
|
|
|
|
|
"""
|
|
|
|
|
self.remove_path_query()
|
2012-11-13 17:11:25 +00:00
|
|
|
if "status/" in self.path:
|
|
|
|
|
self.send_response(self.get_status())
|
|
|
|
|
self.end_headers()
|
|
|
|
|
else:
|
|
|
|
|
super(NoQueryHttpRequestHandler, self).do_HEAD()
|
2005-12-07 21:55:16 +00:00
|
|
|
|
2012-10-15 12:36:27 +00:00
|
|
|
def list_directory(self, path):
|
|
|
|
|
"""Helper to produce a directory listing (absent index.html).
|
|
|
|
|
|
|
|
|
|
Return value is either a file object, or None (indicating an
|
|
|
|
|
error). In either case, the headers are sent, making the
|
|
|
|
|
interface the same as for send_head().
|
|
|
|
|
|
|
|
|
|
"""
|
2019-09-15 18:49:33 +00:00
|
|
|
f = BytesIO()
|
|
|
|
|
f.write(b'<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
|
|
|
|
|
f.write(b"<html>\n<title>Dummy directory listing</title>\n")
|
|
|
|
|
f.write(b"<body>\n<h2>Dummy test directory listing</h2>\n")
|
|
|
|
|
f.write(b"<hr>\n<ul>\n")
|
2020-04-30 19:11:59 +00:00
|
|
|
list = ["example1.txt", "example2.html", "example3"]
|
2012-10-15 12:36:27 +00:00
|
|
|
for name in list:
|
|
|
|
|
displayname = linkname = name
|
2020-05-28 19:29:13 +00:00
|
|
|
list_item = '<li><a href="%s">%s</a>\n' % (
|
|
|
|
|
urllib.parse.quote(linkname),
|
|
|
|
|
html.escape(displayname),
|
2019-09-15 18:49:33 +00:00
|
|
|
)
|
|
|
|
|
f.write(list_item.encode())
|
|
|
|
|
f.write(b"</ul>\n<hr>\n</body>\n</html>\n")
|
2012-10-15 12:36:27 +00:00
|
|
|
length = f.tell()
|
|
|
|
|
f.seek(0)
|
|
|
|
|
self.send_response(200)
|
|
|
|
|
encoding = "utf-8"
|
|
|
|
|
self.send_header("Content-type", "text/html; charset=%s" % encoding)
|
|
|
|
|
self.send_header("Content-Length", str(length))
|
|
|
|
|
self.end_headers()
|
|
|
|
|
return f
|
|
|
|
|
|
2005-12-07 21:55:16 +00:00
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
class HttpServerTest(LinkCheckTest):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
Start/stop an HTTP server that can be used for testing.
|
|
|
|
|
"""
|
|
|
|
|
|
2020-05-28 19:29:13 +00:00
|
|
|
def __init__(self, methodName="runTest"):
|
2005-12-07 21:55:16 +00:00
|
|
|
"""
|
|
|
|
|
Init test class and store default http server port.
|
|
|
|
|
"""
|
|
|
|
|
super(HttpServerTest, self).__init__(methodName=methodName)
|
2011-05-28 17:24:38 +00:00
|
|
|
self.port = None
|
2012-11-06 20:34:22 +00:00
|
|
|
self.handler = NoQueryHttpRequestHandler
|
2005-12-07 21:55:16 +00:00
|
|
|
|
2012-11-06 20:34:22 +00:00
|
|
|
def setUp(self):
|
2011-05-28 17:24:38 +00:00
|
|
|
"""Start a new HTTP server in a new thread."""
|
2012-11-06 20:34:22 +00:00
|
|
|
self.port = start_server(self.handler)
|
2011-05-28 17:24:38 +00:00
|
|
|
assert self.port is not None
|
2005-12-07 21:55:16 +00:00
|
|
|
|
2012-11-06 20:34:22 +00:00
|
|
|
def tearDown(self):
|
2011-05-28 17:24:38 +00:00
|
|
|
"""Send QUIT request to http server."""
|
|
|
|
|
stop_server(self.port)
|
2005-12-07 21:55:16 +00:00
|
|
|
|
2012-11-06 20:34:22 +00:00
|
|
|
def get_url(self, filename):
|
|
|
|
|
"""Get HTTP URL for filename."""
|
2020-04-30 19:11:59 +00:00
|
|
|
return "http://localhost:%d/tests/checker/data/%s" % (self.port, filename)
|
2012-11-06 20:34:22 +00:00
|
|
|
|
|
|
|
|
|
2019-11-11 20:12:25 +00:00
|
|
|
class HttpsServerTest(HttpServerTest):
|
|
|
|
|
"""
|
|
|
|
|
Start/stop an HTTPS server that can be used for testing.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def setUp(self):
|
|
|
|
|
"""Start a new HTTPS server in a new thread."""
|
|
|
|
|
self.port = start_server(self.handler, https=True)
|
|
|
|
|
assert self.port is not None
|
|
|
|
|
|
|
|
|
|
def tearDown(self):
|
|
|
|
|
"""Send QUIT request to http server."""
|
|
|
|
|
stop_server(self.port, https=True)
|
|
|
|
|
|
|
|
|
|
def get_url(self, filename):
|
|
|
|
|
"""Get HTTP URL for filename."""
|
2020-04-30 19:11:59 +00:00
|
|
|
return "https://localhost:%d/tests/checker/data/%s" % (self.port, filename)
|
2019-11-11 20:12:25 +00:00
|
|
|
|
2005-12-07 21:55:16 +00:00
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def start_server(handler, https=False):
|
2011-05-28 17:24:38 +00:00
|
|
|
"""Start an HTTP server thread and return its port number."""
|
2020-05-28 19:29:13 +00:00
|
|
|
server_address = ("localhost", 0)
|
2005-12-07 21:55:16 +00:00
|
|
|
handler.protocol_version = "HTTP/1.0"
|
2011-05-28 17:24:38 +00:00
|
|
|
httpd = StoppableHttpServer(server_address, handler)
|
2019-11-11 20:12:25 +00:00
|
|
|
if https:
|
2020-05-28 19:29:13 +00:00
|
|
|
httpd.socket = ssl.wrap_socket(
|
|
|
|
|
httpd.socket,
|
2019-11-11 20:12:25 +00:00
|
|
|
keyfile=get_file("https_key.pem"),
|
2020-05-28 19:29:13 +00:00
|
|
|
certfile=get_file("https_cert.pem"),
|
|
|
|
|
server_side=True,
|
|
|
|
|
)
|
2011-05-28 17:24:38 +00:00
|
|
|
port = httpd.server_port
|
|
|
|
|
t = threading.Thread(None, httpd.serve_forever)
|
|
|
|
|
t.start()
|
|
|
|
|
# wait for server to start up
|
|
|
|
|
while True:
|
|
|
|
|
try:
|
2019-11-11 20:12:25 +00:00
|
|
|
if https:
|
2020-05-28 19:29:13 +00:00
|
|
|
conn = HTTPSConnection(
|
|
|
|
|
"localhost:%d" % port, context=ssl._create_unverified_context()
|
|
|
|
|
)
|
2019-11-11 20:12:25 +00:00
|
|
|
else:
|
|
|
|
|
conn = HTTPConnection("localhost:%d" % port)
|
2011-05-28 17:24:38 +00:00
|
|
|
conn.request("GET", "/")
|
|
|
|
|
conn.getresponse()
|
|
|
|
|
break
|
|
|
|
|
except:
|
|
|
|
|
time.sleep(0.5)
|
|
|
|
|
return port
|
|
|
|
|
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def stop_server(port, https=False):
|
2011-05-28 17:24:38 +00:00
|
|
|
"""Stop an HTTP server thread."""
|
2019-11-11 20:12:25 +00:00
|
|
|
if https:
|
2020-05-28 19:29:13 +00:00
|
|
|
conn = HTTPSConnection(
|
|
|
|
|
"localhost:%d" % port, context=ssl._create_unverified_context()
|
|
|
|
|
)
|
2019-11-11 20:12:25 +00:00
|
|
|
else:
|
|
|
|
|
conn = HTTPConnection("localhost:%d" % port)
|
2011-05-28 17:24:38 +00:00
|
|
|
conn.request("QUIT", "/")
|
|
|
|
|
conn.getresponse()
|
2012-09-21 18:34:05 +00:00
|
|
|
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def get_cookie(maxage=2000):
|
2012-09-21 18:34:05 +00:00
|
|
|
data = (
|
|
|
|
|
("Comment", "justatest"),
|
|
|
|
|
("Max-Age", "%d" % maxage),
|
|
|
|
|
("Path", "/"),
|
|
|
|
|
("Version", "1"),
|
|
|
|
|
("Foo", "Bar"),
|
|
|
|
|
)
|
|
|
|
|
return "; ".join('%s="%s"' % (key, value) for key, value in data)
|
|
|
|
|
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
class CookieRedirectHttpRequestHandler(NoQueryHttpRequestHandler):
|
2012-09-21 18:34:05 +00:00
|
|
|
"""Handler redirecting certain requests, and setting cookies."""
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def end_headers(self):
|
2012-09-21 18:34:05 +00:00
|
|
|
"""Send cookie before ending headers."""
|
|
|
|
|
self.send_header("Set-Cookie", get_cookie())
|
|
|
|
|
self.send_header("Set-Cookie", get_cookie(maxage=0))
|
|
|
|
|
super(CookieRedirectHttpRequestHandler, self).end_headers()
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def redirect(self):
|
2012-09-21 18:34:05 +00:00
|
|
|
"""Redirect request."""
|
|
|
|
|
path = self.path.replace("redirect", "newurl")
|
|
|
|
|
self.send_response(302)
|
|
|
|
|
self.send_header("Location", path)
|
|
|
|
|
self.end_headers()
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def redirect_newhost(self):
|
2012-09-21 18:34:05 +00:00
|
|
|
"""Redirect request to a new host."""
|
|
|
|
|
path = "http://www.example.com/"
|
|
|
|
|
self.send_response(302)
|
|
|
|
|
self.send_header("Location", path)
|
|
|
|
|
self.end_headers()
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def redirect_newscheme(self):
|
2012-09-21 18:34:05 +00:00
|
|
|
"""Redirect request to a new scheme."""
|
|
|
|
|
if "file" in self.path:
|
2014-02-28 23:12:34 +00:00
|
|
|
path = "file:README.md"
|
2012-09-21 18:34:05 +00:00
|
|
|
else:
|
|
|
|
|
path = "ftp://example.com/"
|
|
|
|
|
self.send_response(302)
|
|
|
|
|
self.send_header("Location", path)
|
|
|
|
|
self.end_headers()
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def do_GET(self):
|
2012-09-21 18:34:05 +00:00
|
|
|
"""Handle redirections for GET."""
|
|
|
|
|
if "redirect_newscheme" in self.path:
|
|
|
|
|
self.redirect_newscheme()
|
|
|
|
|
elif "redirect_newhost" in self.path:
|
|
|
|
|
self.redirect_newhost()
|
|
|
|
|
elif "redirect" in self.path:
|
|
|
|
|
self.redirect()
|
|
|
|
|
else:
|
|
|
|
|
super(CookieRedirectHttpRequestHandler, self).do_GET()
|
|
|
|
|
|
2020-05-16 19:19:42 +00:00
|
|
|
def do_HEAD(self):
|
2012-09-21 18:34:05 +00:00
|
|
|
"""Handle redirections for HEAD."""
|
|
|
|
|
if "redirect_newscheme" in self.path:
|
|
|
|
|
self.redirect_newscheme()
|
|
|
|
|
elif "redirect_newhost" in self.path:
|
|
|
|
|
self.redirect_newhost()
|
|
|
|
|
elif "redirect" in self.path:
|
|
|
|
|
self.redirect()
|
|
|
|
|
else:
|
|
|
|
|
super(CookieRedirectHttpRequestHandler, self).do_HEAD()
|
2020-04-19 18:05:55 +00:00
|
|
|
|
2020-05-28 19:29:13 +00:00
|
|
|
|
2020-04-19 18:05:55 +00:00
|
|
|
class CGIHandler(CGIHTTPRequestHandler, StoppableHttpRequestHandler):
|
|
|
|
|
cgi_path = "/tests/checker/cgi-bin/"
|
|
|
|
|
|
|
|
|
|
def is_cgi(self):
|
|
|
|
|
# CGIHTTPRequestHandler.is_cgi() can only handle a single-level path
|
|
|
|
|
# override so that we can store scripts under /tests/checker
|
|
|
|
|
if CGIHandler.cgi_path in self.path:
|
2020-05-28 19:29:13 +00:00
|
|
|
self.cgi_info = (
|
|
|
|
|
CGIHandler.cgi_path,
|
|
|
|
|
os.path.relpath(self.path, CGIHandler.cgi_path),
|
|
|
|
|
)
|
2020-04-19 18:05:55 +00:00
|
|
|
return True
|
|
|
|
|
return False
|
