mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-03-30 12:40:33 +00:00
12 lines
312 B
Text
12 lines
312 B
Text
|
# Simple robots.txt example, put this into your web root.
|
||
|
|
# See the complete reference at
|
||
|
|
# http://www.robotstxt.org/wc/norobots-rfc.html
|
||
|
|
|
||
|
|
# disallow cgi-bin access for all robots
|
||
|
|
User-agent: *
|
||
|
|
Disallow: /cgi-bin/
|
||
|
|
|
||
|
|
# All LinkChecker versions are not allowed to check anything
|
||
|
|
User-agent: LinkChecker
|
||
|
|
Disallow: /
|