diff --git a/linkcheck/tests/test_robotparser.py b/linkcheck/tests/test_robotparser.py index 3961230c..83d8f7c6 100644 --- a/linkcheck/tests/test_robotparser.py +++ b/linkcheck/tests/test_robotparser.py @@ -96,6 +96,13 @@ class TestRobotParser (unittest.TestCase): self.check(self.rp.can_fetch('Mozilla', 'http://www.lycos.com/search'), True) + def test_password_robots (self): + # whole site is password-protected. + self.rp.set_url('http://mueblesmoraleda.com/robots.txt') + self.rp.read() + self.check(self.rp.can_fetch("*", + "http://mueblesmoraleda.com/"), False) + def test_suite (): """