From 1f5a1e92d3bcd76a23971990848800987173884f Mon Sep 17 00:00:00 2001 From: Chris Mayo Date: Wed, 30 Nov 2022 19:21:06 +0000 Subject: [PATCH 1/6] Fix VirusCheck.check() docstring --- linkcheck/plugins/viruscheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkcheck/plugins/viruscheck.py b/linkcheck/plugins/viruscheck.py index 6adbdfef..1db079e1 100644 --- a/linkcheck/plugins/viruscheck.py +++ b/linkcheck/plugins/viruscheck.py @@ -40,7 +40,7 @@ class VirusCheck(_ContentPlugin): return self.clamav_conf and not url_data.extern[0] def check(self, url_data): - """Try to ask GeoIP database for country info.""" + """Scan content for viruses.""" data = url_data.get_raw_content() infected, errors = scan(data, self.clamav_conf) if infected or errors: From 56e5e94acd81453a49bd3f87fba1e1a6fd354997 Mon Sep 17 00:00:00 2001 From: Chris Mayo Date: Wed, 30 Nov 2022 19:21:06 +0000 Subject: [PATCH 2/6] Improve sqlite3 names in configuration.Modules --- linkcheck/configuration/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkcheck/configuration/__init__.py b/linkcheck/configuration/__init__.py index f940c5ba..cdbce82d 100644 --- a/linkcheck/configuration/__init__.py +++ b/linkcheck/configuration/__init__.py @@ -72,8 +72,8 @@ Modules = ( ("argcomplete", "Argcomplete", None), ("GeoIP", "GeoIP", 'lib_version'), # on Unix systems ("pygeoip", "GeoIP", 'lib_version'), # on Windows systems - ("sqlite3", "Pysqlite", 'version'), - ("sqlite3", "Sqlite", 'sqlite_version'), + ("sqlite3", "sqlite3", 'version'), + ("sqlite3", "SQLite", 'sqlite_version'), ("meliae", "Meliae", '__version__'), ) From a9312f39c48f6bdc62d1a8c5330e3529d6627e5d Mon Sep 17 00:00:00 2001 From: Chris Mayo Date: Wed, 30 Nov 2022 19:21:06 +0000 Subject: [PATCH 3/6] Document installing requirements from distribution packages --- doc/install.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/install.txt b/doc/install.txt index c02057dd..f4d65071 100644 --- a/doc/install.txt +++ b/doc/install.txt @@ -74,6 +74,13 @@ the ``linkchecker`` package is available for installation. pip_ will be available, often as a package e.g. ``python3-pip``, to install the latest LinkChecker. +You may wish to install your distribution's copies of LinkChecker's dependencies +before using pip to install LinkChecker. e.g. for Debian/Ubuntu: + +``apt install python3-bs4 python3-dnspython python3-requests`` + +If those packages are too old pip will install newer versions. + Manual setup for Unix systems ----------------------------- First, install the required software. From 2294160a6a424858d89331b51027abd4ba757c7b Mon Sep 17 00:00:00 2001 From: Chris Mayo Date: Wed, 30 Nov 2022 19:21:06 +0000 Subject: [PATCH 4/6] Fix minimum version of Beautiful Soup increased to 4.11.0 Since: 6d9061b0 ("Ignore bs4 markup and XML parser warnings", 2022-09-02) --- linkcheck/htmlutil/htmlsoup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/linkcheck/htmlutil/htmlsoup.py b/linkcheck/htmlutil/htmlsoup.py index d8af6a08..34886a32 100644 --- a/linkcheck/htmlutil/htmlsoup.py +++ b/linkcheck/htmlutil/htmlsoup.py @@ -28,11 +28,13 @@ warnings.filterwarnings( import bs4 -warnings.simplefilter( +# bs4 4.9.1 introduced MarkupResemblesLocatorWarning +hasattr(bs4, "MarkupResemblesLocatorWarning") and warnings.simplefilter( 'ignore', bs4.MarkupResemblesLocatorWarning ) -warnings.simplefilter( +# bs4 4.11.0 introduced builder.XMLParsedAsHTMLWarning +hasattr(bs4.builder, "XMLParsedAsHTMLWarning") and warnings.simplefilter( 'ignore', bs4.builder.XMLParsedAsHTMLWarning ) From f489660925fc5500debe9ab37cfc2d516f8cbba1 Mon Sep 17 00:00:00 2001 From: Chris Mayo Date: Wed, 30 Nov 2022 19:21:06 +0000 Subject: [PATCH 5/6] Test with minimum versions of requirements Pick 1.8.1 as the minimum version of argcomplete. This is the version in the current Debian oldstable and Ubuntu 20.04 LTS. --- .github/workflows/build.yml | 7 +++++-- pytest-minreqs.ini | 4 ++++ requirements-min.txt | 6 ++++++ requirements.txt | 2 +- tox.ini | 8 +++++--- 5 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 pytest-minreqs.ini create mode 100644 requirements-min.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 386b7d97..dd453758 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,11 +20,14 @@ jobs: strategy: matrix: python-version: - - "3.7" - "3.8" - "3.9" - "3.10" - "3.11" + toxenv: [py] + include: + - python-version: "3.7" + toxenv: minreqs steps: - name: Install OS dependencies @@ -69,7 +72,7 @@ jobs: - name: Run tests run: | python -m hatchling build -t sdist --hooks-only - python -m tox -e py + python -m tox -e ${{ matrix.toxenv }} - name: Report to coveralls run: coveralls diff --git a/pytest-minreqs.ini b/pytest-minreqs.ini new file mode 100644 index 00000000..09c5d952 --- /dev/null +++ b/pytest-minreqs.ini @@ -0,0 +1,4 @@ +# This file is needed until minimum bs4 >= 4.11 +[pytest] +testpaths = tests +addopts = -ra --tb=short diff --git a/requirements-min.txt b/requirements-min.txt new file mode 100644 index 00000000..ce6b6701 --- /dev/null +++ b/requirements-min.txt @@ -0,0 +1,6 @@ +# required: +beautifulsoup4 == 4.8.1 +requests == 2.4.0 +dnspython == 2.0.0 +# optional: +argcomplete == 1.8.1 diff --git a/requirements.txt b/requirements.txt index 9774675f..b65656f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ beautifulsoup4 >= 4.8.1 requests >= 2.4 dnspython >= 2.0 # optional: -argcomplete +argcomplete >= 1.8.1 diff --git a/tox.ini b/tox.ini index 55651d31..c4a414a9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, py311 +envlist = py3{7,8,9,10,11}, minreqs [base] deps = @@ -15,10 +15,12 @@ deps = [testenv] usedevelop = true deps = - -rrequirements.txt + !minreqs: -rrequirements.txt + minreqs: -rrequirements-min.txt {[base]deps} commands = - pytest {posargs:--cov=linkcheck} + !minreqs: pytest {posargs:--cov=linkcheck} + minreqs: pytest {posargs:--cov=linkcheck -c pytest-minreqs.ini} setenv = LC_ALL=en_US.utf-8 From f35448f8ef549f2f7d05034d31be3d76be49610b Mon Sep 17 00:00:00 2001 From: Chris Mayo Date: Wed, 30 Nov 2022 19:21:06 +0000 Subject: [PATCH 6/6] Increase minimum Requests version to 2.19 With Python 3.7 and Requests 2.4.0 on Ubuntu 20.04 there are two test failures; reasons not immediately clear. There is a warning for the use of ABCs from collections and not collections.abc. That will cause an exception on Python 3.10 and is fixed in Requests 2.19.0. --- pyproject.toml | 2 +- requirements-min.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b1b8292d..41d22d40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ requires-python = ">=3.7" dependencies = [ - "requests >= 2.4", + "requests >= 2.19", "dnspython >= 2.0", "beautifulsoup4 >= 4.8.1", ] diff --git a/requirements-min.txt b/requirements-min.txt index ce6b6701..b2b2f144 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -1,6 +1,6 @@ # required: beautifulsoup4 == 4.8.1 -requests == 2.4.0 +requests == 2.19.0 dnspython == 2.0.0 # optional: argcomplete == 1.8.1 diff --git a/requirements.txt b/requirements.txt index b65656f9..c12f5530 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # required: beautifulsoup4 >= 4.8.1 -requests >= 2.4 +requests >= 2.19 dnspython >= 2.0 # optional: argcomplete >= 1.8.1