added robots.txt test

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@1583 e7d03fd6-7b0d-0410-9947-9c21f3af8025

linkcheck/ftests/data/norobots.html

@@ -0,0 +1,2 @@
+<meta name="robots" content="nofollow">
+<a href="do_not_check.html">bla</a>

linkcheck/ftests/data/norobots.html.result

@@ -0,0 +1,4 @@
+url 'file://%(curdir)s/linkcheck/ftests/data/norobots.html'
+cache key file://%(curdir)s/linkcheck/ftests/data/norobots.html
+real url file://%(curdir)s/linkcheck/ftests/data/norobots.html
+valid

linkcheck/ftests/test_robotstxt.py

@@ -0,0 +1,41 @@
+# -*- coding: iso-8859-1 -*-
+"""test HTML robots.txt parsing"""
+# Copyright (C) 2004  Bastian Kleineidam
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+import unittest
+import os
+
+import linkcheck.ftests
+
+
+class TestRobotsTxt (linkcheck.ftests.StandardTest):
+    """test robots.txt directive parsing in HTML files"""
+
+    def test_norobot (self):
+        """test links of norobot.html"""
+        self.file_test("norobots.html")
+
+
+def test_suite ():
+    """build and return a TestSuite"""
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(TestRobotsTxt))
+    return suite
+
+
+if __name__ == '__main__':
+    unittest.main()