From bb8bca1b70ac7d798b9f0c1161f9f9ba8a7245f2 Mon Sep 17 00:00:00 2001
From: calvin <calvin@e7d03fd6-7b0d-0410-9947-9c21f3af8025>
Date: Sun, 13 Jul 2008 13:01:59 +0000
Subject: [PATCH] Add non-ascii values to test robots.txt

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3851 e7d03fd6-7b0d-0410-9947-9c21f3af8025
---
 robots.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/robots.txt b/robots.txt
index a87c96c9..610fa6bf 100644
--- a/robots.txt
+++ b/robots.txt
@@ -6,6 +6,8 @@
 User-agent: LinkChecker
 Crawl-delay: 1
 Disallow: /secret
+# a comment line inside a rule, followed by non-ascii
+Disallow: /umlaut/ö
 
 # The following rule should not match
 User-agent: *