Move URL retreiving functions into url.py module.

linkcheck/gui/checker.py

@@ -41,7 +41,6 @@ class CheckerThread (QtCore.QThread):
     def cancel (self):
         # stop checking
         if self.progress is not None:
-            self.progress.cancelButton.setEnabled(False)
             self.progress = None
         if self.aggregate is not None:
             self.aggregate.wanted_stop = True

linkcheck/gui/linkchecker_ui_progress.py

@@ -2,8 +2,8 @@
 
 # Form implementation generated from reading ui file 'ui/progress.ui'
 #
-# Created: Wed Mar  4 21:31:01 2009
-#      by: PyQt4 UI code generator 4.4.4
+# Created: Fri Oct  1 19:28:45 2010
+#      by: PyQt4 UI code generator 4.7.3
 #
 # WARNING! All changes made in this file will be lost!
 
@@ -13,42 +13,40 @@ class Ui_ProgressDialog(object):
     def setupUi(self, ProgressDialog):
         ProgressDialog.setObjectName("ProgressDialog")
         ProgressDialog.setWindowModality(QtCore.Qt.ApplicationModal)
-        ProgressDialog.resize(322, 217)
-        self.cancelButton = QtGui.QPushButton(ProgressDialog)
-        self.cancelButton.setGeometry(QtCore.QRect(240, 180, 70, 27))
-        self.cancelButton.setObjectName("cancelButton")
+        ProgressDialog.resize(289, 185)
+        self.verticalLayout = QtGui.QVBoxLayout(ProgressDialog)
+        self.verticalLayout.setObjectName("verticalLayout")
         self.frame = QtGui.QFrame(ProgressDialog)
-        self.frame.setGeometry(QtCore.QRect(10, 10, 301, 161))
         self.frame.setFrameShape(QtGui.QFrame.StyledPanel)
         self.frame.setFrameShadow(QtGui.QFrame.Raised)
         self.frame.setObjectName("frame")
-        self.progressBar = QtGui.QProgressBar(self.frame)
-        self.progressBar.setGeometry(QtCore.QRect(10, 120, 281, 23))
-        self.progressBar.setProperty("value", QtCore.QVariant(24))
-        self.progressBar.setObjectName("progressBar")
+        self.verticalLayout_2 = QtGui.QVBoxLayout(self.frame)
+        self.verticalLayout_2.setObjectName("verticalLayout_2")
         self.tabWidget = QtGui.QTabWidget(self.frame)
-        self.tabWidget.setGeometry(QtCore.QRect(10, 10, 281, 101))
         self.tabWidget.setObjectName("tabWidget")
         self.tab = QtGui.QWidget()
         self.tab.setObjectName("tab")
-        self.label = QtGui.QLabel(self.tab)
-        self.label.setGeometry(QtCore.QRect(10, 10, 48, 17))
-        self.label.setObjectName("label")
+        self.formLayout = QtGui.QFormLayout(self.tab)
+        self.formLayout.setFieldGrowthPolicy(QtGui.QFormLayout.ExpandingFieldsGrow)
+        self.formLayout.setObjectName("formLayout")
         self.label_active = QtGui.QLabel(self.tab)
-        self.label_active.setGeometry(QtCore.QRect(80, 10, 48, 17))
         self.label_active.setObjectName("label_active")
+        self.formLayout.setWidget(0, QtGui.QFormLayout.FieldRole, self.label_active)
         self.label_2 = QtGui.QLabel(self.tab)
-        self.label_2.setGeometry(QtCore.QRect(10, 30, 48, 17))
         self.label_2.setObjectName("label_2")
+        self.formLayout.setWidget(1, QtGui.QFormLayout.LabelRole, self.label_2)
         self.label_queued = QtGui.QLabel(self.tab)
-        self.label_queued.setGeometry(QtCore.QRect(80, 30, 48, 17))
         self.label_queued.setObjectName("label_queued")
+        self.formLayout.setWidget(1, QtGui.QFormLayout.FieldRole, self.label_queued)
         self.label_3 = QtGui.QLabel(self.tab)
-        self.label_3.setGeometry(QtCore.QRect(10, 50, 48, 17))
         self.label_3.setObjectName("label_3")
+        self.formLayout.setWidget(2, QtGui.QFormLayout.LabelRole, self.label_3)
         self.label_checked = QtGui.QLabel(self.tab)
-        self.label_checked.setGeometry(QtCore.QRect(80, 50, 48, 17))
         self.label_checked.setObjectName("label_checked")
+        self.formLayout.setWidget(2, QtGui.QFormLayout.FieldRole, self.label_checked)
+        self.label = QtGui.QLabel(self.tab)
+        self.label.setObjectName("label")
+        self.formLayout.setWidget(0, QtGui.QFormLayout.LabelRole, self.label)
         self.tabWidget.addTab(self.tab, "")
         self.tab_2 = QtGui.QWidget()
         self.tab_2.setObjectName("tab_2")
@@ -57,27 +55,51 @@ class Ui_ProgressDialog(object):
         self.scrollArea.setWidgetResizable(True)
         self.scrollArea.setObjectName("scrollArea")
         self.scrollAreaWidgetContents = QtGui.QWidget(self.scrollArea)
-        self.scrollAreaWidgetContents.setGeometry(QtCore.QRect(0, 0, 267, 67))
+        self.scrollAreaWidgetContents.setGeometry(QtCore.QRect(0, 0, 265, 65))
         self.scrollAreaWidgetContents.setObjectName("scrollAreaWidgetContents")
         self.textBrowser = QtGui.QTextBrowser(self.scrollAreaWidgetContents)
         self.textBrowser.setGeometry(QtCore.QRect(0, 0, 271, 71))
         self.textBrowser.setObjectName("textBrowser")
         self.scrollArea.setWidget(self.scrollAreaWidgetContents)
         self.tabWidget.addTab(self.tab_2, "")
+        self.verticalLayout_2.addWidget(self.tabWidget)
+        self.progressBar = QtGui.QProgressBar(self.frame)
+        self.progressBar.setProperty("value", 24)
+        self.progressBar.setObjectName("progressBar")
+        self.verticalLayout_2.addWidget(self.progressBar)
+        self.widget = QtGui.QWidget(self.frame)
+        self.widget.setObjectName("widget")
+        self.horizontalLayout = QtGui.QHBoxLayout(self.widget)
+        self.horizontalLayout.setObjectName("horizontalLayout")
+        self.cancelLabel = QtGui.QLabel(self.widget)
+        self.cancelLabel.setText("")
+        self.cancelLabel.setObjectName("cancelLabel")
+        self.horizontalLayout.addWidget(self.cancelLabel)
+        self.cancelButton = QtGui.QPushButton(self.widget)
+        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
+        sizePolicy.setHorizontalStretch(0)
+        sizePolicy.setVerticalStretch(0)
+        sizePolicy.setHeightForWidth(self.cancelButton.sizePolicy().hasHeightForWidth())
+        self.cancelButton.setSizePolicy(sizePolicy)
+        self.cancelButton.setObjectName("cancelButton")
+        self.horizontalLayout.addWidget(self.cancelButton)
+        self.verticalLayout_2.addWidget(self.widget)
+        self.verticalLayout.addWidget(self.frame)
+        self.cancelLabel.setBuddy(self.cancelButton)
 
         self.retranslateUi(ProgressDialog)
-        self.tabWidget.setCurrentIndex(1)
+        self.tabWidget.setCurrentIndex(0)
         QtCore.QMetaObject.connectSlotsByName(ProgressDialog)
 
     def retranslateUi(self, ProgressDialog):
         ProgressDialog.setWindowTitle(_("LinkChecker progress"))
-        self.cancelButton.setText(_("Cancel"))
-        self.label.setText(_("Active:"))
         self.label_active.setText(_("0"))
         self.label_2.setText(_("Queued:"))
         self.label_queued.setText(_("0"))
         self.label_3.setText(_("Checked:"))
         self.label_checked.setText(_("0"))
+        self.label.setText(_("Active:"))
         self.tabWidget.setTabText(self.tabWidget.indexOf(self.tab), _("Status"))
         self.tabWidget.setTabText(self.tabWidget.indexOf(self.tab_2), _("Debug"))
+        self.cancelButton.setText(_("Cancel"))
 

linkcheck/gui/progress.py

@@ -38,6 +38,7 @@ class LinkCheckerProgress (QtGui.QDialog, Ui_ProgressDialog):
         self.progressBar.setMaximum(0)
         set_fixed_font(self.textBrowser)
         self.connect(self, QtCore.SIGNAL("log_status(int,int,int,float)"), self.log_status)
+        self.connect(self.cancelButton, QtCore.SIGNAL("clicked()"), self.cancel)
 
     def log_status (self, checked, in_progress, queued, duration):
         self.label_checked.setText(u"%d" % checked)
@@ -56,6 +57,11 @@ class LinkCheckerProgress (QtGui.QDialog, Ui_ProgressDialog):
         self.label_active.setText(u"0")
         self.label_queued.setText(u"0")
         self.label_checked.setText(u"0")
+        self.cancelLabel.setText(u"")
+
+    def cancel (self):
+        self.cancelButton.setEnabled(False)
+        self.cancelLabel.setText(_(u"Closing pending connections..."))
 
 
 class StatusLogger (object):

linkcheck/gui/ui/progress.ui

@@ -1,196 +1,171 @@
-<ui version="4.0" >
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
  <class>ProgressDialog</class>
- <widget class="QDialog" name="ProgressDialog" >
-  <property name="windowModality" >
+ <widget class="QDialog" name="ProgressDialog">
+  <property name="windowModality">
    <enum>Qt::ApplicationModal</enum>
   </property>
-  <property name="geometry" >
+  <property name="geometry">
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>322</width>
-    <height>217</height>
+    <width>289</width>
+    <height>185</height>
    </rect>
   </property>
-  <property name="windowTitle" >
+  <property name="windowTitle">
    <string>LinkChecker progress</string>
   </property>
-  <widget class="QPushButton" name="cancelButton" >
-   <property name="geometry" >
-    <rect>
-     <x>240</x>
-     <y>180</y>
-     <width>70</width>
-     <height>27</height>
-    </rect>
-   </property>
-   <property name="text" >
-    <string>Cancel</string>
-   </property>
-  </widget>
-  <widget class="QFrame" name="frame" >
-   <property name="geometry" >
-    <rect>
-     <x>10</x>
-     <y>10</y>
-     <width>301</width>
-     <height>161</height>
-    </rect>
-   </property>
-   <property name="frameShape" >
-    <enum>QFrame::StyledPanel</enum>
-   </property>
-   <property name="frameShadow" >
-    <enum>QFrame::Raised</enum>
-   </property>
-   <widget class="QProgressBar" name="progressBar" >
-    <property name="geometry" >
-     <rect>
-      <x>10</x>
-      <y>120</y>
-      <width>281</width>
-      <height>23</height>
-     </rect>
-    </property>
-    <property name="value" >
-     <number>24</number>
-    </property>
-   </widget>
-   <widget class="QTabWidget" name="tabWidget" >
-    <property name="geometry" >
-     <rect>
-      <x>10</x>
-      <y>10</y>
-      <width>281</width>
-      <height>101</height>
-     </rect>
-    </property>
-    <property name="currentIndex" >
-     <number>1</number>
-    </property>
-    <widget class="QWidget" name="tab" >
-     <attribute name="title" >
-      <string>Status</string>
-     </attribute>
-     <widget class="QLabel" name="label" >
-      <property name="geometry" >
-       <rect>
-        <x>10</x>
-        <y>10</y>
-        <width>48</width>
-        <height>17</height>
-       </rect>
-      </property>
-      <property name="text" >
-       <string>Active:</string>
-      </property>
-     </widget>
-     <widget class="QLabel" name="label_active" >
-      <property name="geometry" >
-       <rect>
-        <x>80</x>
-        <y>10</y>
-        <width>48</width>
-        <height>17</height>
-       </rect>
-      </property>
-      <property name="text" >
-       <string>0</string>
-      </property>
-     </widget>
-     <widget class="QLabel" name="label_2" >
-      <property name="geometry" >
-       <rect>
-        <x>10</x>
-        <y>30</y>
-        <width>48</width>
-        <height>17</height>
-       </rect>
-      </property>
-      <property name="text" >
-       <string>Queued:</string>
-      </property>
-     </widget>
-     <widget class="QLabel" name="label_queued" >
-      <property name="geometry" >
-       <rect>
-        <x>80</x>
-        <y>30</y>
-        <width>48</width>
-        <height>17</height>
-       </rect>
-      </property>
-      <property name="text" >
-       <string>0</string>
-      </property>
-     </widget>
-     <widget class="QLabel" name="label_3" >
-      <property name="geometry" >
-       <rect>
-        <x>10</x>
-        <y>50</y>
-        <width>48</width>
-        <height>17</height>
-       </rect>
-      </property>
-      <property name="text" >
-       <string>Checked:</string>
-      </property>
-     </widget>
-     <widget class="QLabel" name="label_checked" >
-      <property name="geometry" >
-       <rect>
-        <x>80</x>
-        <y>50</y>
-        <width>48</width>
-        <height>17</height>
-       </rect>
-      </property>
-      <property name="text" >
-       <string>0</string>
-      </property>
-     </widget>
-    </widget>
-    <widget class="QWidget" name="tab_2" >
-     <attribute name="title" >
-      <string>Debug</string>
-     </attribute>
-     <widget class="QScrollArea" name="scrollArea" >
-      <property name="geometry" >
-       <rect>
-        <x>0</x>
-        <y>0</y>
-        <width>271</width>
-        <height>71</height>
-       </rect>
-      </property>
-      <property name="widgetResizable" >
-       <bool>true</bool>
-      </property>
-      <widget class="QWidget" name="scrollAreaWidgetContents" >
-       <property name="geometry" >
-        <rect>
-         <x>0</x>
-         <y>0</y>
-         <width>267</width>
-         <height>67</height>
-        </rect>
-       </property>
-       <widget class="QTextBrowser" name="textBrowser" >
-        <property name="geometry" >
-         <rect>
-          <x>0</x>
-          <y>0</y>
-          <width>271</width>
-          <height>71</height>
-         </rect>
+  <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <widget class="QFrame" name="frame">
+     <property name="frameShape">
+      <enum>QFrame::StyledPanel</enum>
+     </property>
+     <property name="frameShadow">
+      <enum>QFrame::Raised</enum>
+     </property>
+     <layout class="QVBoxLayout" name="verticalLayout_2">
+      <item>
+       <widget class="QTabWidget" name="tabWidget">
+        <property name="currentIndex">
+         <number>0</number>
+        </property>
+        <widget class="QWidget" name="tab">
+         <attribute name="title">
+          <string>Status</string>
+         </attribute>
+         <layout class="QFormLayout" name="formLayout">
+          <property name="fieldGrowthPolicy">
+           <enum>QFormLayout::ExpandingFieldsGrow</enum>
+          </property>
+          <item row="0" column="1">
+           <widget class="QLabel" name="label_active">
+            <property name="text">
+             <string>0</string>
+            </property>
+           </widget>
+          </item>
+          <item row="1" column="0">
+           <widget class="QLabel" name="label_2">
+            <property name="text">
+             <string>Queued:</string>
+            </property>
+            <property name="buddy">
+             <cstring></cstring>
+            </property>
+           </widget>
+          </item>
+          <item row="1" column="1">
+           <widget class="QLabel" name="label_queued">
+            <property name="text">
+             <string>0</string>
+            </property>
+           </widget>
+          </item>
+          <item row="2" column="0">
+           <widget class="QLabel" name="label_3">
+            <property name="text">
+             <string>Checked:</string>
+            </property>
+           </widget>
+          </item>
+          <item row="2" column="1">
+           <widget class="QLabel" name="label_checked">
+            <property name="text">
+             <string>0</string>
+            </property>
+           </widget>
+          </item>
+          <item row="0" column="0">
+           <widget class="QLabel" name="label">
+            <property name="text">
+             <string>Active:</string>
+            </property>
+           </widget>
+          </item>
+         </layout>
+        </widget>
+        <widget class="QWidget" name="tab_2">
+         <attribute name="title">
+          <string>Debug</string>
+         </attribute>
+         <widget class="QScrollArea" name="scrollArea">
+          <property name="geometry">
+           <rect>
+            <x>0</x>
+            <y>0</y>
+            <width>271</width>
+            <height>71</height>
+           </rect>
+          </property>
+          <property name="widgetResizable">
+           <bool>true</bool>
+          </property>
+          <widget class="QWidget" name="scrollAreaWidgetContents">
+           <property name="geometry">
+            <rect>
+             <x>0</x>
+             <y>0</y>
+             <width>265</width>
+             <height>65</height>
+            </rect>
+           </property>
+           <widget class="QTextBrowser" name="textBrowser">
+            <property name="geometry">
+             <rect>
+              <x>0</x>
+              <y>0</y>
+              <width>271</width>
+              <height>71</height>
+             </rect>
+            </property>
+           </widget>
+          </widget>
+         </widget>
+        </widget>
+       </widget>
+      </item>
+      <item>
+       <widget class="QProgressBar" name="progressBar">
+        <property name="value">
+         <number>24</number>
         </property>
        </widget>
-      </widget>
-     </widget>
+      </item>
+      <item>
+       <widget class="QWidget" name="widget" native="true">
+        <layout class="QHBoxLayout" name="horizontalLayout">
+         <item>
+          <widget class="QLabel" name="cancelLabel">
+           <property name="text">
+            <string/>
+           </property>
+           <property name="buddy">
+            <cstring>cancelButton</cstring>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QPushButton" name="cancelButton">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="text">
+            <string>Cancel</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </widget>
+      </item>
+     </layout>
     </widget>
-   </widget>
-  </widget>
+   </item>
+  </layout>
  </widget>
  <resources/>
  <connections/>

linkcheck/robotparser2.py

@@ -27,36 +27,12 @@ import time
 import socket
 import sys
 from . import httplib2 as httplib
-from . import log, LOG_CHECK, configuration, httputil
+from . import url as urlutil
+from . import log, LOG_CHECK, configuration
 
 __all__ = ["RobotFileParser"]
 
 
-class PasswordManager (object):
-    """Simple password manager storing username and password. Suitable
-    for use as an AuthHandler instance in urllib2."""
-
-    def __init__ (self, user, password):
-        """Store given username and password."""
-        self.user = user
-        self.password = password
-
-    def add_password (self, realm, uri, user, passwd):
-        """Does nothing since username and password are already stored.
-
-        @return: None
-        """
-        pass
-
-    def find_user_password (self, realm, authuri):
-        """Get stored username and password.
-
-        @return: A tuple (user, password)
-        @rtype: tuple
-        """
-        return self.user, self.password
-
-
 class RobotFileParser (object):
     """This class provides a set of methods to read, parse and answer
     questions about a single robots.txt file."""
@@ -99,35 +75,6 @@ class RobotFileParser (object):
         self.url = url
         self.host, self.path = urlparse.urlparse(url)[1:3]
 
-    def get_opener (self):
-        """Construct an URL opener object. It considers the given credentials
-        from the __init__() method and supports proxies.
-
-        @return: URL opener
-        @rtype: urllib2.OpenerDirector
-        """
-        pwd_manager = PasswordManager(self.user, self.password)
-        handlers = [
-            urllib2.UnknownHandler,
-            httputil.HttpWithGzipHandler,
-            urllib2.HTTPBasicAuthHandler(pwd_manager),
-            urllib2.HTTPDigestAuthHandler(pwd_manager),
-        ]
-        if self.proxy:
-            handlers.insert(0,
-              urllib2.ProxyHandler({"http": self.proxy, "https": self.proxy}))
-            handlers.extend([
-                urllib2.ProxyBasicAuthHandler(pwd_manager),
-                urllib2.ProxyDigestAuthHandler(pwd_manager),
-            ])
-        handlers.extend([
-            urllib2.HTTPDefaultErrorHandler,
-            urllib2.HTTPRedirectHandler,
-        ])
-        if hasattr(httplib, 'HTTPS'):
-            handlers.append(httputil.HttpsWithGzipHandler)
-        return urllib2.build_opener(*handlers)
-
     def read (self):
         """Read the robots.txt URL and feeds it to the parser."""
         self._reset()
@@ -177,13 +124,16 @@ class RobotFileParser (object):
         @raise: httplib.HTTPException, IOError on HTTP errors
         @raise: ValueError on bad digest auth (a bug)
         """
-        f = self.get_opener().open(req)
-        ct = f.info().get("Content-Type")
-        if ct and ct.lower().startswith("text/plain"):
-            self.parse([line.strip() for line in f])
-        else:
-            self.allow_all = True
-        f.close()
+        f = urlutil.get_opener(self.user, self.password, self.proxy)
+        try:
+            f.open(req)
+            ct = f.info().get("Content-Type")
+            if ct and ct.lower().startswith("text/plain"):
+                self.parse([line.strip() for line in f])
+            else:
+                self.allow_all = True
+        finally:
+            f.close()
 
     def _add_entry (self, entry):
         """Add a parsed entry to entry list.

linkcheck/url.py

@@ -22,6 +22,9 @@ import re
 import os
 import urlparse
 import urllib
+import urllib2
+import socket
+from . import httplib2 as httplib
 
 urlparse.uses_netloc.extend(('ldap', 'irc'))
 
@@ -476,3 +479,85 @@ def splitport (host, port=80):
         if iport:
             port = iport
     return host, port
+
+
+class PasswordManager (object):
+    """Simple password manager storing username and password. Suitable
+    for use as an AuthHandler instance in urllib2."""
+
+    def __init__ (self, user, password):
+        """Store given username and password."""
+        self.user = user
+        self.password = password
+
+    def add_password (self, realm, uri, user, passwd):
+        """Does nothing since username and password are already stored.
+
+        @return: None
+        """
+        pass
+
+    def find_user_password (self, realm, authuri):
+        """Get stored username and password.
+
+        @return: A tuple (user, password)
+        @rtype: tuple
+        """
+        return self.user, self.password
+
+
+def get_opener (user=None, password=None, proxy=None):
+    """Construct an URL opener object. It considers the given credentials
+    and proxy.
+
+    @return: URL opener
+    @rtype: urllib2.OpenerDirector
+    """
+    from . import httputil
+    pwd_manager = PasswordManager(user, password)
+    handlers = [
+        urllib2.UnknownHandler,
+        httputil.HttpWithGzipHandler,
+        urllib2.HTTPBasicAuthHandler(pwd_manager),
+        urllib2.HTTPDigestAuthHandler(pwd_manager),
+    ]
+    if proxy:
+        handlers.insert(0,
+          urllib2.ProxyHandler({"http": proxy, "https": proxy}))
+        handlers.extend([
+            urllib2.ProxyBasicAuthHandler(pwd_manager),
+            urllib2.ProxyDigestAuthHandler(pwd_manager),
+        ])
+    handlers.extend([
+        urllib2.HTTPDefaultErrorHandler,
+        urllib2.HTTPRedirectHandler,
+    ])
+    if hasattr(httplib, 'HTTPS'):
+        handlers.append(httputil.HttpsWithGzipHandler)
+    return urllib2.build_opener(*handlers)
+
+
+def get_content (url, user=None, password=None, proxy=None):
+    """Get URL content and info.
+
+    @return: (url info, content), or (None, None) on error.
+    @rtype: tuple (string, string)
+    """
+    from . import configuration
+    headers = {
+        'User-Agent': configuration.UserAgent,
+        'Accept-Encoding' : 'gzip;q=1.0, deflate;q=0.9, identity;q=0.5',
+    }
+    req = urllib2.Request(url, None, headers)
+    try:
+        f = get_opener(user=user, password=password, proxy=proxy)
+        try:
+            f.open(req)
+            return (f.info(), f.read())
+        finally:
+            f.close()
+    except (urllib2.HTTPError, socket.timeout, urllib2.URLError,
+            socket.gaierror, socket.error, IOError, httplib.HTTPException,
+            ValueError):
+        pass
+    return (None, None)