diff --git a/doc/changelog.txt b/doc/changelog.txt index 8a37a39b..1c913753 100644 --- a/doc/changelog.txt +++ b/doc/changelog.txt @@ -3,6 +3,8 @@ Fixes: - checking: Do not remove CGI parameters when joining URLs. - checking: Correctly detect empty FTP paths as directories. +- checking: Reuse connections more than once and ensure they are + closed before expiring. - gui: Remove old context menu action to view URL properties. - gui: Disable viewing of parent URL source if it's a directory. diff --git a/linkcheck/cache/connection.py b/linkcheck/cache/connection.py index 1407c384..fc5adb34 100644 --- a/linkcheck/cache/connection.py +++ b/linkcheck/cache/connection.py @@ -65,10 +65,12 @@ class ConnectionPool (object): def add (self, key, conn, timeout): """Add connection to the pool with given identifier key and timeout in seconds.""" + assert timeout >= 1.0, 'invalid timeout value %r' % timeout self.connections[key] = [conn, 'available', time.time() + timeout] @synchronized(_wait_lock) def wait_for_host (self, host): + """Honor wait time for given host.""" t = time.time() if host in self.times: due_time = self.times[host] @@ -95,16 +97,18 @@ class ConnectionPool (object): return None conn_data = self.connections[key] t = time.time() - if t > conn_data[2]: - # timed out + if (t+5.0) >= conn_data[2]: + # already or soon to be timed out self._remove_connection(key) return None if conn_data[1] == 'busy': # connection is in use return None - # mark busy and return + log.debug(LOG_CACHE, + "reusing connection %s timing out in %.01f seconds", + key, (conn_data[2] - t)) + # mark busy and return connection object conn_data[1] = 'busy' - conn_data[2] = t return conn_data[0] @synchronized(_lock) @@ -115,11 +119,11 @@ class ConnectionPool (object): @synchronized(_lock) def remove_expired (self): - """Remove expired connections from this pool.""" + """Remove expired or soon to be expired connections from this pool.""" t = time.time() to_delete = [] for key, conn_data in self.connections.items(): - if conn_data[1] == 'available' and t > conn_data[2]: + if conn_data[1] == 'available' and (t+5.0) >= conn_data[2]: to_delete.append(key) for key in to_delete: self._remove_connection(key)