mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-03-16 22:10:26 +00:00
bb8bca1b70
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3851 e7d03fd6-7b0d-0410-9947-9c21f3af8025
14 lines
387 B
Text
14 lines
387 B
Text
# A test robots.txt file. It resides in the root directory to be found
|
|
# by the unittest HTTP server.
|
|
# Note that while unit tests are running, the contents of this file
|
|
# can change.
|
|
|
|
User-agent: LinkChecker
|
|
Crawl-delay: 1
|
|
Disallow: /secret
|
|
# a comment line inside a rule, followed by non-ascii
|
|
Disallow: /umlaut/ö
|
|
|
|
# The following rule should not match
|
|
User-agent: *
|
|
Crawl-delay: 100
|