add get_crawldelay method

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3226 e7d03fd6-7b0d-0410-9947-9c21f3af8025

linkcheck/robotparser2.py

@@ -357,6 +357,18 @@ class RobotFileParser (object):
         # agent not found ==> access granted
         return True
 
+    def get_crawldelay (self, useragent):
+        """
+        Look for a configured crawl delay.
+
+        @return: crawl delay in seconds or zero
+        @rtype: integer >= 0
+        """
+        for entry in self.entries:
+            if entry.applies_to(useragent):
+                return entry.crawldelay
+        return 0
+
     def __str__ (self):
         """
         Constructs string representation, usable as contents of a