Enable https checking using a test server

Verification has to be turned off because we are using a
self-signed certificate.

dev-requirements.txt

@@ -3,6 +3,7 @@ miniboa
 parameterized
 pdfminer
 pyftpdlib
+pyopenssl
 pytest
 pytest-xdist
 # for building on osx:

tests/checker/httpserver.py

@@ -20,7 +20,8 @@ Define http test support classes for LinkChecker tests.
 
 from html import escape as html_escape
 from http.server import SimpleHTTPRequestHandler, HTTPServer
-from http.client import HTTPConnection
+from http.client import HTTPConnection, HTTPSConnection
+import ssl
 import time
 import threading
 try:
@@ -29,6 +30,7 @@ except ImportError:
     import urllib as urllib_parse
 from io import BytesIO
 from . import LinkCheckTest
+from .. import get_file
 
 
 class StoppableHttpRequestHandler (SimpleHTTPRequestHandler, object):
@@ -175,19 +177,45 @@ class HttpServerTest (LinkCheckTest):
         return u"http://localhost:%d/tests/checker/data/%s" % (self.port, filename)
 
 
+class HttpsServerTest(HttpServerTest):
+    """
+    Start/stop an HTTPS server that can be used for testing.
+    """
 
-def start_server (handler):
+    def setUp(self):
+        """Start a new HTTPS server in a new thread."""
+        self.port = start_server(self.handler, https=True)
+        assert self.port is not None
+
+    def tearDown(self):
+        """Send QUIT request to http server."""
+        stop_server(self.port, https=True)
+
+    def get_url(self, filename):
+        """Get HTTP URL for filename."""
+        return u"https://localhost:%d/tests/checker/data/%s" % (self.port, filename)
+
+
+def start_server (handler, https=False):
     """Start an HTTP server thread and return its port number."""
     server_address = ('localhost', 0)
     handler.protocol_version = "HTTP/1.0"
     httpd = StoppableHttpServer(server_address, handler)
+    if https:
+        httpd.socket = ssl.wrap_socket(httpd.socket,
+            keyfile=get_file("https_key.pem"),
+            certfile=get_file("https_cert.pem"), server_side=True)
     port = httpd.server_port
     t = threading.Thread(None, httpd.serve_forever)
     t.start()
     # wait for server to start up
     while True:
         try:
-            conn = HTTPConnection("localhost:%d" % port)
+            if https:
+                conn = HTTPSConnection("localhost:%d" % port,
+                        context=ssl._create_unverified_context())
+            else:
+                conn = HTTPConnection("localhost:%d" % port)
             conn.request("GET", "/")
             conn.getresponse()
             break
@@ -196,9 +224,13 @@ def start_server (handler):
     return port
 
 
-def stop_server (port):
+def stop_server (port, https=False):
     """Stop an HTTP server thread."""
-    conn = HTTPConnection("localhost:%d" % port)
+    if https:
+        conn = HTTPSConnection("localhost:%d" % port,
+                               context=ssl._create_unverified_context())
+    else:
+        conn = HTTPConnection("localhost:%d" % port)
     conn.request("QUIT", "/")
     conn.getresponse()
 

tests/checker/test_https.py

@@ -15,34 +15,51 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 """
-Test news checking.
+Test https.
 """
-import pytest
+from OpenSSL import crypto
 
 from tests import need_network
-from . import LinkCheckTest
+from .httpserver import HttpsServerTest, CookieRedirectHttpRequestHandler
+from .. import get_file
 
 
-class TestHttps (LinkCheckTest):
+class TestHttps(HttpsServerTest):
     """
     Test https: link checking.
     """
 
-    @pytest.mark.xfail
+    def __init__(self, methodName='runTest'):
+        super(TestHttps, self).__init__(methodName=methodName)
+        self.handler = CookieRedirectHttpRequestHandler
+
+    @classmethod
+    def setUpClass(cls):
+        key = crypto.PKey()
+        key.generate_key(crypto.TYPE_RSA, 2048)
+        cert = crypto.X509()
+        cert.get_subject().CN = "localhost"
+        cert.set_serial_number(1000)
+        cert.gmtime_adj_notBefore(0)
+        cert.set_notAfter(b"21190102030405Z")
+        cert.set_issuer(cert.get_subject())
+        cert.set_pubkey(key)
+        cert.sign(key, 'sha1')
+        with open(get_file("https_key.pem"), "wb") as f:
+            f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, key))
+        with open(get_file("https_cert.pem"), "wb") as f:
+            f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
+
     @need_network
     def test_https (self):
-        url = u"https://www.amazon.com/"
-        rurl = u"https://www.amazon.com/"
+        url = self.get_url("")
         resultlines = [
             u"url %s" % url,
             u"cache key %s" % url,
-            u"real url %s" % rurl,
-            #u"info SSL cipher RC4-SHA, TLSv1/SSLv3.",
-            u"info Access denied by robots.txt, checked only syntax.",
+            u"real url %s" % url,
             u"valid",
         ]
         confargs = dict(
-            #enabledplugins=['SslCertificateCheck'],
-            #SslCertificateCheck=dict(sslcertwarndays=10),
+            sslverify=False
         )
         self.direct(url, resultlines, recursionlevel=0, confargs=confargs)

tox.ini

@@ -9,6 +9,7 @@ deps =
     parameterized
     py27: pdfminer < 20191010
     !py27: pdfminer
+    pyopenssl
     pytest-xdist
     pytest-cov
     miniboa