mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-04-21 22:54:45 +00:00
13 lines
312 B
Text
13 lines
312 B
Text
|
# A test robots.txt file. It resides in the root directory to be found
|
||
|
|
# by the unittest HTTP server.
|
||
|
|
# Note that while unit tests are running, the contents of this file
|
||
|
|
# can change.
|
||
|
|
|
||
|
|
User-agent: LinkChecker
|
||
|
|
Crawl-delay: 1
|
||
|
|
Disallow: /secret
|
||
|
|
|
||
|
|
# The following rule should not match
|
||
|
|
User-agent: *
|
||
|
|
Crawl-delay: 100
|