add charset tests

tests/checker/data/charsets/cp1250.html

@@ -0,0 +1,3 @@
+<meta http-equiv="Content-Type" content="text/html; charset=cp1250">
+<a href="file.html">Žlu<EFBFBD>ouèký kúò úpìl ïábelské ódy ùùùù</a>
+<img src="img.png" alt="Žlu<6C>ouèký kúò úpìl ïábelské ódy ùùùù"/>

tests/checker/data/charsets/cp1250.html.result

@@ -0,0 +1,17 @@
+url file://%(curdir)s/%(datadir)s/charsets/cp1250.html
+cache key file://%(curdir)s/%(datadir)s/charsets/cp1250.html
+real url file://%(curdir)s/%(datadir)s/charsets/cp1250.html
+name %(datadir)s/charsets/cp1250.html
+valid
+
+url file.html
+cache key file://%(curdir)s/%(datadir)s/charsets/file.html
+real url file://%(curdir)s/%(datadir)s/charsets/file.html
+name Žluťoučký kúň úpěl ďábelské ódy ůůůů
+error
+
+url img.png
+cache key file://%(curdir)s/%(datadir)s/charsets/img.png
+real url file://%(curdir)s/%(datadir)s/charsets/img.png
+name Žluťoučký kúň úpěl ďábelské ódy ůůůů
+error

tests/checker/data/charsets/iso8859-2.html

@@ -0,0 +1,3 @@
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-2">
+<a href="file.html">®lu»ouèký kúò úpìl ïábelské ódy ùùùù</a>
+<img src="img.png" alt="®lu»ouèký kúò úpìl ïábelské ódy ùùùù"/>

tests/checker/data/charsets/iso8859-2.html.result

@@ -0,0 +1,17 @@
+url file://%(curdir)s/%(datadir)s/charsets/iso8859-2.html
+cache key file://%(curdir)s/%(datadir)s/charsets/iso8859-2.html
+real url file://%(curdir)s/%(datadir)s/charsets/iso8859-2.html
+name %(datadir)s/charsets/iso8859-2.html
+valid
+
+url file.html
+cache key file://%(curdir)s/%(datadir)s/charsets/file.html
+real url file://%(curdir)s/%(datadir)s/charsets/file.html
+name Žluťoučký kúň úpěl ďábelské ódy ůůůů
+error
+
+url img.png
+cache key file://%(curdir)s/%(datadir)s/charsets/img.png
+real url file://%(curdir)s/%(datadir)s/charsets/img.png
+name Žluťoučký kúň úpěl ďábelské ódy ůůůů
+error

tests/checker/data/charsets/utf8.html

@@ -0,0 +1,3 @@
+<meta charset="utf-8">
+<a href="file.html">Žluťoučký kúň úpěl ďábelské ódy ůůůů</a>
+<img src="img.png" alt="Žluťoučký kúň úpěl ďábelské ódy ůůůů"/>

tests/checker/data/charsets/utf8.html.result

@@ -0,0 +1,17 @@
+url file://%(curdir)s/%(datadir)s/charsets/utf8.html
+cache key file://%(curdir)s/%(datadir)s/charsets/utf8.html
+real url file://%(curdir)s/%(datadir)s/charsets/utf8.html
+name %(datadir)s/charsets/utf8.html
+valid
+
+url file.html
+cache key file://%(curdir)s/%(datadir)s/charsets/file.html
+real url file://%(curdir)s/%(datadir)s/charsets/file.html
+name Žluťoučký kúň úpěl ďábelské ódy ůůůů
+error
+
+url img.png
+cache key file://%(curdir)s/%(datadir)s/charsets/img.png
+real url file://%(curdir)s/%(datadir)s/charsets/img.png
+name Žluťoučký kúň úpěl ďábelské ódy ůůůů
+error

tests/checker/test_charsets.py

@@ -0,0 +1,39 @@
+# Copyright (C) 2004-2014 Bastian Kleineidam
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""
+Test html <base> tag parsing.
+"""
+from . import LinkCheckTest
+
+
+class TestBase (LinkCheckTest):
+    """
+    Test, if charset encoding is done right.
+    The linkchecker should translate the encoding
+    from the original source and show it on the terminal
+    in most readable form.
+    Check the tested files with browser - the link text
+    should look the same in all of them.
+    """
+
+    def test_utf8(self):
+        self.file_test("charsets/utf8.html")
+
+    def test_iso8859_2(self):
+        self.file_test("charsets/iso8859-2.html")
+
+    def test_cp1250(self):
+        self.file_test("charsets/cp1250.html")