From c57f607fc3b728126d702799f1831f3c7cf58e96 Mon Sep 17 00:00:00 2001 From: Bastian Kleineidam Date: Mon, 7 Apr 2014 18:54:33 +0200 Subject: [PATCH] Use urldata.add_url() --- linkcheck/cache/robots_txt.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/linkcheck/cache/robots_txt.py b/linkcheck/cache/robots_txt.py index 0bb17422..3396c4fa 100644 --- a/linkcheck/cache/robots_txt.py +++ b/linkcheck/cache/robots_txt.py @@ -21,7 +21,6 @@ from .. import robotparser2, configuration from ..containers import LFUCache from ..decorators import synchronized from ..lock import get_lock -from ..checker import get_url_from # lock objects @@ -76,11 +75,7 @@ class RobotsTxt (object): if rec_level >= 0 and url_data.recursion_level >= rec_level: return for sitemap_url, line in rp.sitemap_urls: - sitemap_url_data = get_url_from(sitemap_url, - url_data.recursion_level+1, url_data.aggregate, - parent_url=roboturl, line=line, - parent_content_type=url_data.content_type) - url_data.aggregate.urlqueue.put(sitemap_url_data) + url_data.add_url(sitemap_url, line=line) @synchronized(robot_lock) def get_lock(self, roboturl):