mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-05-02 11:54:43 +00:00
commit
a56b8d6456
11 changed files with 20 additions and 53 deletions
|
|
@ -18,11 +18,7 @@ Handle FTP links.
|
|||
"""
|
||||
|
||||
import ftplib
|
||||
try:
|
||||
from cStringIO import StringIO
|
||||
except ImportError:
|
||||
# Python 3
|
||||
from io import StringIO
|
||||
from io import StringIO
|
||||
|
||||
from .. import log, LOG_CHECK, LinkCheckerError, mimeutil
|
||||
from . import proxysupport, httpurl, internpaturl, get_index_html
|
||||
|
|
|
|||
|
|
@ -15,11 +15,9 @@
|
|||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
"""Parse configuration files"""
|
||||
|
||||
try: # Python 3
|
||||
from configparser import RawConfigParser
|
||||
except ImportError: # Python 2
|
||||
from ConfigParser import RawConfigParser
|
||||
from configparser import RawConfigParser
|
||||
import os
|
||||
|
||||
from .. import LinkCheckerError, get_link_pat, LOG_CHECK, log, fileutil, plugins, logconf
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,10 +17,7 @@
|
|||
Parsing of cookies.
|
||||
"""
|
||||
|
||||
try: # Python 3
|
||||
from http.cookiejar import split_header_words
|
||||
except ImportError: # Python 2
|
||||
from cookielib import split_header_words
|
||||
from http.cookiejar import split_header_words
|
||||
import email
|
||||
import requests
|
||||
|
||||
|
|
|
|||
|
|
@ -17,11 +17,8 @@
|
|||
Management of checking a queue of links with several threads.
|
||||
"""
|
||||
import os
|
||||
try: # Python 3
|
||||
from _thread import error as thread_error
|
||||
except ImportError: # Python 2
|
||||
from thread import error as thread_error
|
||||
import time
|
||||
|
||||
from .. import log, LOG_CHECK, LinkCheckerInterrupt, plugins
|
||||
from ..cache import urlqueue, robots_txt, results
|
||||
from . import aggregator, console
|
||||
|
|
@ -54,7 +51,7 @@ def check_urls (aggregate):
|
|||
raise
|
||||
except KeyboardInterrupt:
|
||||
interrupt(aggregate)
|
||||
except thread_error:
|
||||
except RuntimeError:
|
||||
log.warn(LOG_CHECK,
|
||||
_("Could not start a new thread. Check that the current user" \
|
||||
" is allowed to start new threads."))
|
||||
|
|
|
|||
|
|
@ -17,10 +17,6 @@
|
|||
Aggregate needed object instances for checker threads.
|
||||
"""
|
||||
import threading
|
||||
try: # Python 3
|
||||
import _thread
|
||||
except ImportError:
|
||||
import thread as _thread
|
||||
|
||||
import requests
|
||||
import time
|
||||
|
|
@ -126,19 +122,19 @@ class Aggregate:
|
|||
self.threads.append(t)
|
||||
t.start()
|
||||
else:
|
||||
self.request_sessions[_thread.get_ident()] = new_request_session(self.config, self.cookies)
|
||||
self.request_sessions[threading.get_ident()] = new_request_session(self.config, self.cookies)
|
||||
checker.check_urls(self.urlqueue, self.logger)
|
||||
|
||||
@synchronized(_threads_lock)
|
||||
def add_request_session(self):
|
||||
"""Add a request session for current thread."""
|
||||
session = new_request_session(self.config, self.cookies)
|
||||
self.request_sessions[_thread.get_ident()] = session
|
||||
self.request_sessions[threading.get_ident()] = session
|
||||
|
||||
@synchronized(_threads_lock)
|
||||
def get_request_session(self):
|
||||
"""Get the request session for current thread."""
|
||||
return self.request_sessions[_thread.get_ident()]
|
||||
return self.request_sessions[threading.get_ident()]
|
||||
|
||||
@synchronized(_hosts_lock)
|
||||
def wait_for_host(self, host):
|
||||
|
|
|
|||
|
|
@ -15,10 +15,8 @@
|
|||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
"""Logger for aggregator instances"""
|
||||
import threading
|
||||
try: # Python 3
|
||||
import _thread
|
||||
except ImportError: # Python 2
|
||||
import thread as _thread
|
||||
import _thread
|
||||
|
||||
from ..decorators import synchronized
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
|
|
|||
|
|
@ -13,10 +13,8 @@
|
|||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
try: # Python 3
|
||||
import _thread
|
||||
except ImportError: # Python 2
|
||||
import thread as _thread
|
||||
import _thread
|
||||
|
||||
from ..decorators import notimplemented
|
||||
from .. import threader
|
||||
from . import console
|
||||
|
|
|
|||
|
|
@ -17,15 +17,11 @@
|
|||
Logging and debug functions.
|
||||
"""
|
||||
|
||||
from io import StringIO
|
||||
import logging
|
||||
import os
|
||||
import inspect
|
||||
import traceback
|
||||
try:
|
||||
from cStringIO import StringIO
|
||||
except ImportError:
|
||||
# Python 3
|
||||
from io import StringIO
|
||||
|
||||
# memory leak debugging
|
||||
#import gc
|
||||
|
|
|
|||
|
|
@ -17,11 +17,9 @@
|
|||
A CSV logger.
|
||||
"""
|
||||
import csv
|
||||
from io import StringIO
|
||||
import os
|
||||
try:
|
||||
from cStringIO import StringIO
|
||||
except ImportError: # Python 3
|
||||
from io import StringIO
|
||||
|
||||
from . import _Logger
|
||||
from .. import strformat
|
||||
|
||||
|
|
|
|||
|
|
@ -16,11 +16,9 @@
|
|||
"""
|
||||
Test proxy handling.
|
||||
"""
|
||||
from test.support import EnvironmentVarGuard
|
||||
|
||||
from . import httpserver
|
||||
try:
|
||||
from test.support import EnvironmentVarGuard
|
||||
except ImportError: # Python 3
|
||||
from test.test_support import EnvironmentVarGuard
|
||||
|
||||
class TestProxy (httpserver.HttpServerTest):
|
||||
"""Test no_proxy env var handling."""
|
||||
|
|
|
|||
|
|
@ -17,14 +17,9 @@
|
|||
Test decorators.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from io import StringIO
|
||||
import time
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO
|
||||
except ImportError:
|
||||
# Python 3
|
||||
from io import StringIO
|
||||
import unittest
|
||||
|
||||
import linkcheck.decorators
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue