mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-03-22 17:00:25 +00:00
4ec74f6f5c
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3232 e7d03fd6-7b0d-0410-9947-9c21f3af8025
12 lines
312 B
Text
12 lines
312 B
Text
# A test robots.txt file. It resides in the root directory to be found
|
|
# by the unittest HTTP server.
|
|
# Note that while unit tests are running, the contents of this file
|
|
# can change.
|
|
|
|
User-agent: LinkChecker
|
|
Crawl-delay: 1
|
|
Disallow: /secret
|
|
|
|
# The following rule should not match
|
|
User-agent: *
|
|
Crawl-delay: 100
|