added robots.txt parse test

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3212 e7d03fd6-7b0d-0410-9947-9c21f3af8025

linkcheck/tests/test_robotstxt.py

@@ -0,0 +1,68 @@
+# -*- coding: iso-8859-1 -*-
+# Copyright (C) 2006 Bastian Kleineidam
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+"""
+Test robots.txt parsing.
+"""
+
+import unittest
+import linkcheck.robotparser2
+
+
+class TestRobotsTxt (unittest.TestCase):
+    """
+    Test string formatting routines.
+    """
+
+    def setUp (self):
+        """
+        Initialize self.rp as a robots.txt parser.
+        """
+        self.rp = linkcheck.robotparser2.RobotFileParser()
+
+    def test_robotstxt (self):
+        lines = [
+            "User-agent: *",
+        ]
+        self.rp.parse(lines)
+        self.assertEquals(str(self.rp), "\n".join(lines))
+
+    def test_robotstxt2 (self):
+        lines = [
+            "User-agent: *",
+            "Disallow: /search",
+        ]
+        self.rp.parse(lines)
+        self.assertEquals(str(self.rp), "\n".join(lines))
+
+    def test_crawldelay (self):
+        lines = [
+            "User-agent: *",
+            "Crawl-delay: 10",
+        ]
+        self.rp.parse(lines)
+        self.assertEquals(str(self.rp), "\n".join(lines))
+
+
+def test_suite ():
+    """
+    Build and return a TestSuite.
+    """
+    return unittest.makeSuite(TestRobotsTxt)
+
+
+if __name__ == '__main__':
+    unittest.main()