mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-03-19 07:20:26 +00:00
77 lines
2.5 KiB
Python
77 lines
2.5 KiB
Python
# -*- coding: iso-8859-1 -*-
|
|
# Copyright (C) 2000-2014 Bastian Kleineidam
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
"""
|
|
Main functions for link parsing
|
|
"""
|
|
from xml.parsers.expat import ParserCreate
|
|
|
|
|
|
class XmlTagUrlParser(object):
|
|
"""Parse XML files and find URLs in text content of a tag name."""
|
|
|
|
def __init__(self, tag):
|
|
"""Initialize the parser."""
|
|
self.tag = tag
|
|
self.parser = ParserCreate()
|
|
self.parser.buffer_text = True
|
|
self.parser.returns_unicode = True
|
|
self.parser.StartElementHandler = self.start_element
|
|
self.parser.EndElementHandler = self.end_element
|
|
self.parser.CharacterDataHandler = self.char_data
|
|
|
|
def parse(self, url_data):
|
|
"""Parse XML URL data."""
|
|
self.url_data = url_data
|
|
self.loc = False
|
|
self.url = u""
|
|
data = url_data.get_content()
|
|
isfinal = True
|
|
self.parser.Parse(data, isfinal)
|
|
|
|
def start_element(self, name, attrs):
|
|
"""Set tag status for start element."""
|
|
self.in_tag = (name == self.tag)
|
|
self.url = u""
|
|
|
|
def end_element(self, name):
|
|
"""If end tag is our tag, call add_url()."""
|
|
self.in_tag = False
|
|
if name == self.tag:
|
|
self.add_url()
|
|
|
|
def add_url(self):
|
|
"""Add non-empty URLs to the queue."""
|
|
if self.url:
|
|
self.url_data.add_url(self.url, line=self.parser.CurrentLineNumber,
|
|
column=self.parser.CurrentColumnNumber)
|
|
self.url = u""
|
|
|
|
def char_data(self, data):
|
|
"""If inside the wanted tag, append data to URL."""
|
|
if self.loc:
|
|
self.url += data
|
|
|
|
|
|
def parse_sitemap(url_data):
|
|
"""Parse XML sitemap data."""
|
|
XmlTagUrlParser(u"loc").parse(url_data)
|
|
|
|
|
|
def parse_sitemapindex(url_data):
|
|
"""Parse XML sitemap index data."""
|
|
XmlTagUrlParser(u"loc").parse(url_data)
|
|
|