Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-05 13:14:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove the unnormed URL warning.

Browse source
This commit is contained in:
Bastian Kleineidam 2010-07-29 20:09:47 +02:00
parent 7ad4f7c220
commit 0f92b76290
14 changed files with 8 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config/linkcheckerrc
View file

@ -161,7 +161,7 @@
# recognized warnings). Add a comma-separated list of warnings here
# that prevent a valid URL from being logged. Note that the warning
# will be logged in invalid URLs.
#ignorewarnings=url-unnormed,url-unicode-domain,anchor-not-found
#ignorewarnings=url-unicode-domain,anchor-not-found
# Regular expression to add more URLs recognized as internal links.
# Default is that URLs given on the command line are internal.

4
doc/changelog.txt
View file

@ -17,12 +17,16 @@ Changes:
- checking: Get size from Content-Length HTTP header, from stat(2)
for local files so size information is available without downloading
the content data.
- checking: Remove the unnormed URL warning. URLs can be written
in more than one way and there is no norm.
Closes: SF bug #1575800
Features:
- ftp: Detect and support UTF-8 filename encoding capability of FTP
servers.
- checking: added new warning to check if content size is zero
5.2 "11:14" (released 7.3.2010)
Fixes:

2
linkcheck/checker/const.py
View file

@ -75,7 +75,6 @@ ExcList = ExcCacheList + ExcNoCacheList
WARN_URL_EFFECTIVE_URL = "url-effective-url"
WARN_URL_ERROR_GETTING_CONTENT = "url-error-getting-content"
WARN_URL_UNICODE_DOMAIN = "url-unicode-domain"
WARN_URL_UNNORMED = "url-unnormed"
WARN_URL_ANCHOR_NOT_FOUND = "url-anchor-not-found"
WARN_URL_WARNREGEX_FOUND = "url-warnregex-found"
WARN_URL_CONTENT_SIZE_TOO_LARGE = "url-content-too-large"
@ -106,7 +105,6 @@ Warnings = {
WARN_URL_ERROR_GETTING_CONTENT:
_("Could not get the content of the URL."),
WARN_URL_UNICODE_DOMAIN: _("URL uses a unicode domain."),
WARN_URL_UNNORMED: _("URL is not normed."),
WARN_URL_ANCHOR_NOT_FOUND: _("URL anchor was not found."),
WARN_URL_WARNREGEX_FOUND:
_("The warning regular expression was found in the URL contents."),

6
linkcheck/checker/urlbase.py
View file

@ -36,7 +36,7 @@ from .. import (log, LOG_CHECK, LOG_CACHE, httputil, httplib2 as httplib,
from ..HtmlParser import htmlsax
from ..htmlutil import linkparse
from .const import (WARN_URL_EFFECTIVE_URL, WARN_URL_UNICODE_DOMAIN,
WARN_URL_UNNORMED, WARN_URL_ERROR_GETTING_CONTENT,
WARN_URL_ERROR_GETTING_CONTENT,
WARN_URL_ANCHOR_NOT_FOUND, WARN_URL_WARNREGEX_FOUND,
WARN_URL_CONTENT_SIZE_TOO_LARGE, WARN_URL_CONTENT_SIZE_ZERO,
WARN_URL_CONTENT_SIZE_UNEQUAL, ExcList, ExcSyntaxList, ExcNoCacheList)
@ -355,10 +355,6 @@ class UrlBase (object):
the URL %(idna_url)r instead.""") % \
{"url": self.base_url, "idna_url": base_url},
tag=WARN_URL_UNICODE_DOMAIN)
elif self.base_url != base_url:
self.add_warning(
_("Base URL is not properly normed. Normed URL is %(url)s.") %
{'url': base_url}, tag=WARN_URL_UNNORMED)
# make url absolute
if self.base_ref:
# use base reference as parent url

8
tests/checker/data/http.html.result
View file

@ -46,14 +46,12 @@ url http:/localhost:8001/
cache key http:///localhost%%3A8001/
real url http:///localhost%%3A8001/
name one slash
warning Base URL is not properly normed. Normed URL is http:///localhost%%3A8001/.
error
url http:localhost:8001/ (cached)
cache key http:///localhost%%3A8001/
real url http:///localhost%%3A8001/
name no slash
warning Base URL is not properly normed. Normed URL is http:///localhost%%3A8001/.
error
url //localhost:8001/ (cached)
@ -72,21 +70,18 @@ url http:/
cache key http:///
real url http:///
name no url, one slash
warning Base URL is not properly normed. Normed URL is http:///.
error
url http: (cached)
cache key http://
real url http://
name no url, no slash
warning Base URL is not properly normed. Normed URL is http://.
error
url http://localhost:8001/stalter&sohn
cache key http://localhost:8001/stalter%%26sohn
real url http://localhost:8001/stalter%%26sohn
name unquoted ampersand
warning Base URL is not properly normed. Normed URL is http://localhost:8001/stalter%%26sohn.
error
url http://slashdot.org/
@ -99,14 +94,12 @@ url HtTP://LoCAlHOst:8001/ (cached)
cache key http://localhost:8001/
real url http://localhost:8001/
name should be cached
warning Base URL is not properly normed. Normed URL is http://localhost:8001/.
valid
url HTTP://LOCALHOST:8001/ (cached)
cache key http://localhost:8001/
real url http://localhost:8001/
name should be cached
warning Base URL is not properly normed. Normed URL is http://localhost:8001/.
valid
url http://localhost:8001/?d=directory&p=page
@ -125,7 +118,6 @@ url http://localhost:8001/?quoted=ü
cache key http://localhost:8001/?quoted=%%C3%%BC
real url http://localhost:8001/?quoted=%%C3%%BC
name html entities
warning Base URL is not properly normed. Normed URL is http://localhost:8001/?quoted=%%C3%%BC.
valid
url clsid:12345

1
tests/checker/data/Мошкова.bin.result
View file

@ -2,6 +2,5 @@ url file://%(curdir)s/%(datadir)s/Мошкова.bin
cache key file://%(curdir)s/%(datadir)s/%%D0%%9C%%D0%%BE%%D1%%88%%D0%%BA%%D0%%BE%%D0%%B2%%D0%%B0.bin
real url file://%(curdir)s/%(datadir)s/%%D0%%9C%%D0%%BE%%D1%%88%%D0%%BA%%D0%%BE%%D0%%B2%%D0%%B0.bin
name %(datadir)s/Мошкова.bin
warning Base URL is not properly normed. Normed URL is file://%(curdir)s/%(datadir)s/%%D0%%9C%%D0%%BE%%D1%%88%%D0%%BA%%D0%%BE%%D0%%B2%%D0%%B0.bin.
valid

8
tests/checker/test_error.py
View file

@ -34,7 +34,6 @@ class TestError (LinkCheckTest):
u"url file://%(curdir)s/%(url)s" % attrs,
u"cache key %(nurl)s" % attrs,
u"real url %(nurl)s" % attrs,
u"warning Base URL is not properly normed. Normed URL is %(nurl)s." % attrs,
u"error",
]
self.direct(url, resultlines)
@ -49,7 +48,6 @@ class TestError (LinkCheckTest):
u"cache key %(nurl)s" % attrs,
u"real url %(nurl)s" % attrs,
u"name %(url)s" % attrs,
u"warning Base URL is not properly normed. Normed URL is %(nurl)s." % attrs,
u"error",
]
self.direct(url, resultlines)
@ -61,7 +59,6 @@ class TestError (LinkCheckTest):
u"cache key %(nurl)s" % attrs,
u"real url %(nurl)s" % attrs,
u"name %(url)s" % attrs,
u"warning Base URL is not properly normed. Normed URL is %(nurl)s." % attrs,
u"error",
]
self.direct(url, resultlines)
@ -74,7 +71,6 @@ class TestError (LinkCheckTest):
u"url %s" % url,
u"cache key %s" % nurl,
u"real url %s" % nurl,
u"warning Base URL is not properly normed. Normed URL is %s." % nurl,
u"error",
]
self.direct(url, resultlines)
@ -84,7 +80,6 @@ class TestError (LinkCheckTest):
u"url %s" % url,
u"cache key %s" % nurl,
u"real url %s" % nurl,
u"warning Base URL is not properly normed. Normed URL is %s." % nurl,
u"error",
]
self.direct(url, resultlines)
@ -99,7 +94,6 @@ class TestError (LinkCheckTest):
u"cache key %(nurl)s" % attrs,
u"real url %(nurl)s" % attrs,
u"name %(url)s" % attrs,
u"warning Base URL is not properly normed. Normed URL is %(nurl)s." % attrs,
u"error",
]
self.direct(url, resultlines)
@ -114,7 +108,6 @@ class TestError (LinkCheckTest):
u"cache key %(nurl)s" % attrs,
u"real url %(nurl)s" % attrs,
u"name %(url)s" % attrs,
u"warning Base URL is not properly normed. Normed URL is %(nurl)s." % attrs,
u"error",
]
self.direct(url, resultlines)
@ -129,7 +122,6 @@ class TestError (LinkCheckTest):
u"cache key %(nurl)s" % attrs,
u"real url %(nurl)s" % attrs,
u"name %(url)s" % attrs,
u"warning Base URL is not properly normed. Normed URL is %(nurl)s." % attrs,
u"error",
]
self.direct(url, resultlines)

1
tests/checker/test_file.py
View file

@ -116,7 +116,6 @@ class TestFile (LinkCheckTest):
u"url %s" % url,
u"cache key file://%(curdir)s/%(datadir)s/file.txt" % attrs,
u"real url file://%(curdir)s/%(datadir)s/file.txt" % attrs,
u"warning Base URL is not properly normed. Normed URL is %s." % nurl,
u"valid",
]
self.direct(url, resultlines)

4
tests/checker/test_ftp.py
View file

@ -44,7 +44,6 @@ class TestFtp (FtpServerTest):
u"url %s" % url,
u"cache key %s" % nurl,
u"real url %s" % nurl,
u"warning Base URL is not properly normed. Normed URL is %s." % nurl,
u"error",
]
self.direct(url, resultlines)
@ -76,7 +75,6 @@ class TestFtp (FtpServerTest):
u"url %s" % url,
u"cache key %s" % nurl,
u"real url %s" % nurl,
u"warning Base URL is not properly normed. Normed URL is %s." % nurl,
u"valid",
]
self.direct(url, resultlines)
@ -87,7 +85,6 @@ class TestFtp (FtpServerTest):
u"url %s" % url,
u"cache key %s" % nurl,
u"real url %s" % nurl,
u"warning Base URL is not properly normed. Normed URL is %s." % nurl,
u"valid",
]
self.direct(url, resultlines)
@ -98,7 +95,6 @@ class TestFtp (FtpServerTest):
u"url %s" % url,
u"cache key %s" % nurl,
u"real url %s" % nurl,
u"warning Base URL is not properly normed. Normed URL is %s." % nurl,
u"error",
]
self.direct(url, resultlines)

4
tests/checker/test_mail.py
View file

@ -103,8 +103,6 @@ class TestMail (LinkCheckTest):
u"cache key mailto:calvin@users.sourceforge.net",
u"real url %s" % qurl,
u"info Verified address calvin@users.sourceforge.net: 250 <calvin@users.sourceforge.net> is deliverable.",
u"warning Base URL is not properly normed. "
u"Normed URL is %s." % qurl,
u"valid",
]
self.direct(url, resultlines)
@ -115,8 +113,6 @@ class TestMail (LinkCheckTest):
u"cache key mailto:calvin@users.sourceforge.net",
u"real url %s" % qurl,
u"info Verified address calvin@users.sourceforge.net: 250 <calvin@users.sourceforge.net> is deliverable.",
u"warning Base URL is not properly normed. "
u"Normed URL is %s." % qurl,
u"valid",
]
self.direct(url, resultlines)

2
tests/checker/test_news.py
View file

@ -66,7 +66,6 @@ class TestNews (LinkCheckTest):
u"url %s" % url,
u"cache key %s" % nurl,
u"real url %s" % nurl,
u"warning Base URL is not properly normed. Normed URL is %s." % nurl,
u"warning No NNTP server was specified, skipping this URL.",
u"valid",
]
@ -81,7 +80,6 @@ class TestNews (LinkCheckTest):
u"url %s" % url,
u"cache key %s" % qurl,
u"real url %s" % qurl,
u"warning Base URL is not properly normed. Normed URL is %s." % qurl,
u"warning No NNTP server was specified, skipping this URL.",
u"valid",
]

1
tests/checker/test_telnet.py
View file

@ -30,7 +30,6 @@ class TestTelnet (LinkCheckTest):
u"url %s" % url,
u"cache key %s" % nurl,
u"real url %s" % nurl,
u"warning Base URL is not properly normed. Normed URL is %s." % nurl,
u"error",
]
self.direct(url, resultlines)

2
tests/configuration/data/config0.ini
View file

@ -21,7 +21,7 @@ nofollow=
nofollow_imadoofus2
nofollow1=nofollow_oldimadoofus1
nofollow2=nofollow_oldimadoofus2
ignorewarnings=url-unnormed,url-unicode-domain,anchor-not-found
ignorewarnings=url-unicode-domain,anchor-not-found
[authentication]
entry=

2
tests/configuration/test_config.py
View file

@ -53,7 +53,7 @@ class TestConfig (unittest.TestCase):
for suffix in ("1", "2"):
key = "%s%simadoofus%s" % (prefix1, prefix2, suffix)
self.assertTrue(key in patterns)
for key in ("url-unnormed", "url-unicode-domain", "anchor-not-found"):
for key in ("url-unicode-domain", "anchor-not-found"):
self.assertTrue(key in config["ignorewarnings"])
# authentication section
patterns = [x["pattern"].pattern for x in config["authentication"]]