Merge pull request #700 from cjmayo/bs4-4.11

Fix minimum bs4 is 4.11 and test minimum requirements

.github/workflows/build.yml

@@ -20,11 +20,14 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"
+        toxenv: [py]
+        include:
+          - python-version: "3.7"
+            toxenv: minreqs
 
     steps:
       - name: Install OS dependencies
@@ -69,7 +72,7 @@ jobs:
       - name: Run tests
         run: |
             python -m hatchling build -t sdist --hooks-only
-            python -m tox -e py
+            python -m tox -e ${{ matrix.toxenv }}
 
       - name: Report to coveralls
         run: coveralls

doc/install.txt

@@ -74,6 +74,13 @@ the ``linkchecker`` package is available for installation. pip_ will be
 available, often as a package e.g. ``python3-pip``, to install the latest
 LinkChecker.
 
+You may wish to install your distribution's copies of LinkChecker's dependencies
+before using pip to install LinkChecker. e.g. for Debian/Ubuntu:
+
+``apt install python3-bs4 python3-dnspython python3-requests``
+
+If those packages are too old pip will install newer versions.
+
 Manual setup for Unix systems
 -----------------------------
 First, install the required software.

linkcheck/configuration/__init__.py

@@ -72,8 +72,8 @@ Modules = (
     ("argcomplete", "Argcomplete", None),
     ("GeoIP", "GeoIP", 'lib_version'),  # on Unix systems
     ("pygeoip", "GeoIP", 'lib_version'),  # on Windows systems
-    ("sqlite3", "Pysqlite", 'version'),
-    ("sqlite3", "Sqlite", 'sqlite_version'),
+    ("sqlite3", "sqlite3", 'version'),
+    ("sqlite3", "SQLite", 'sqlite_version'),
     ("meliae", "Meliae", '__version__'),
 )
 

linkcheck/htmlutil/htmlsoup.py

@@ -28,11 +28,13 @@ warnings.filterwarnings(
 
 import bs4
 
-warnings.simplefilter(
+# bs4 4.9.1 introduced MarkupResemblesLocatorWarning
+hasattr(bs4, "MarkupResemblesLocatorWarning") and warnings.simplefilter(
     'ignore', bs4.MarkupResemblesLocatorWarning
 )
 
-warnings.simplefilter(
+# bs4 4.11.0 introduced builder.XMLParsedAsHTMLWarning
+hasattr(bs4.builder, "XMLParsedAsHTMLWarning") and warnings.simplefilter(
     'ignore', bs4.builder.XMLParsedAsHTMLWarning
 )
 

linkcheck/plugins/viruscheck.py

@@ -40,7 +40,7 @@ class VirusCheck(_ContentPlugin):
         return self.clamav_conf and not url_data.extern[0]
 
     def check(self, url_data):
-        """Try to ask GeoIP database for country info."""
+        """Scan content for viruses."""
         data = url_data.get_raw_content()
         infected, errors = scan(data, self.clamav_conf)
         if infected or errors:

pyproject.toml

@@ -21,7 +21,7 @@ classifiers = [
 requires-python = ">=3.7"
 
 dependencies = [
-    "requests >= 2.4",
+    "requests >= 2.19",
     "dnspython >= 2.0",
     "beautifulsoup4 >= 4.8.1",
 ]

pytest-minreqs.ini

@@ -0,0 +1,4 @@
+# This file is needed until minimum bs4 >= 4.11
+[pytest]
+testpaths = tests
+addopts = -ra --tb=short

requirements-min.txt

@@ -0,0 +1,6 @@
+# required:
+beautifulsoup4 == 4.8.1
+requests == 2.19.0
+dnspython == 2.0.0
+# optional:
+argcomplete == 1.8.1

requirements.txt

@@ -1,6 +1,6 @@
 # required:
 beautifulsoup4 >= 4.8.1
-requests >= 2.4
+requests >= 2.19
 dnspython >= 2.0
 # optional:
-argcomplete
+argcomplete >= 1.8.1

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py37, py38, py39, py310, py311
+envlist = py3{7,8,9,10,11}, minreqs
 
 [base]
 deps =
@@ -15,10 +15,12 @@ deps =
 [testenv]
 usedevelop = true
 deps =
-    -rrequirements.txt
+    !minreqs: -rrequirements.txt
+    minreqs: -rrequirements-min.txt
     {[base]deps}
 commands =
-    pytest {posargs:--cov=linkcheck}
+    !minreqs: pytest {posargs:--cov=linkcheck}
+    minreqs: pytest {posargs:--cov=linkcheck -c pytest-minreqs.ini}
 setenv =
     LC_ALL=en_US.utf-8