Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-04-22 23:24:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

better FTP connection pooling, remove unnecessary LIST

Browse source 
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@2222 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2005-01-28 01:02:15 +00:00
parent 0860aa5e32
commit 7e0c34a3d9
2 changed files with 26 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
ChangeLog
View file

@ -33,6 +33,15 @@
Type: bugfix
Changed: linkcheck/checker/mailtourl.py
* Do not list FTP directory contents when they are not needed.
Type: bugfix
Changed: linkcheck/checker/ftpurl.py
* If an FTP connection is pooled but busy, wait for the connection to
become available for at least 30 seconds.
Type: feature
Changed: linkcheck/checker/ftpurl.py
2.2 "Cube" (released 25.01.2005)
* CSV log format changes:

31
linkcheck/checker/ftpurl.py
View file

@ -44,8 +44,6 @@ class FtpUrl (urlbase.UrlBase, proxysupport.ProxySupport):
self.files = []
# last part of URL filename
self.filename = None
# if FTP connection was reused
self.reuse = False
def check_connection (self):
# proxy support (we support only http)
@ -65,11 +63,7 @@ class FtpUrl (urlbase.UrlBase, proxysupport.ProxySupport):
self.login()
self.filename = self.cwd()
self.listfile()
if self.is_directory():
self.url_connection.cwd(self.filename)
self.files = self.get_files()
else:
self.files = []
self.files = []
return None
def get_user_password (self):
@ -78,18 +72,26 @@ class FtpUrl (urlbase.UrlBase, proxysupport.ProxySupport):
return urllib.splitpasswd(self.userinfo)
return super(FtpUrl, self).get_user_password()
def get_ftp_connection (self, hostport, _user, _password):
cache_get = self.consumer.cache.get_ftp_connection
# wait at most 300*0.1=30 seconds for connection to become available
for dummy in xrange(300):
conn = cache_get(hostport, _user, _password)
if conn != "busy":
return conn
time.sleep(0.1)
return None
def login (self):
"""
Log into ftp server and check the welcome message.
"""
_user, _password = self.get_user_password()
# ready to connect
conn = self.consumer.cache.get_ftp_connection(
self.urlparts[1], _user, _password)
conn = self.get_ftp_connection(self.urlparts[1], _user, _password)
if conn is not None:
# reuse cached FTP connection
self.url_connection = conn
self.reuse = True
return
try:
self.url_connection = ftplib.FTP()
@ -113,7 +115,7 @@ class FtpUrl (urlbase.UrlBase, proxysupport.ProxySupport):
# don't set info anymore, this may change every time we log in
#self.add_info(info)
# add to cached connections
self.reuse = self.consumer.cache.add_ftp_connection(
self.consumer.cache.add_ftp_connection(
self.urlparts[1], _user, _password, self.url_connection)
def cwd (self):
@ -201,6 +203,8 @@ class FtpUrl (urlbase.UrlBase, proxysupport.ProxySupport):
return self.data
t = time.time()
if self.is_directory():
self.url_connection.cwd(self.filename)
self.files = self.get_files()
self.data = linkcheck.checker.get_index_html(self.files)
else:
# download file in BINARY mode
@ -217,8 +221,7 @@ class FtpUrl (urlbase.UrlBase, proxysupport.ProxySupport):
return self.data
def close_connection (self):
if self.reuse:
_user, _password = self.get_user_password()
self.consumer.cache.release_ftp_connection(
_user, _password = self.get_user_password()
self.consumer.cache.release_ftp_connection(
self.urlparts[1], _user, _password)
self.url_connection = None