mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-03-17 06:20:27 +00:00
6840808fa9
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@2217 e7d03fd6-7b0d-0410-9947-9c21f3af8025
11 lines
312 B
Text
11 lines
312 B
Text
# Simple robots.txt example, put this into your web root.
|
|
# See the complete reference at
|
|
# http://www.robotstxt.org/wc/norobots-rfc.html
|
|
|
|
# disallow cgi-bin access for all robots
|
|
User-agent: *
|
|
Disallow: /cgi-bin/
|
|
|
|
# All LinkChecker versions are not allowed to check anything
|
|
User-agent: LinkChecker
|
|
Disallow: /
|