mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-03-17 06:20:27 +00:00
ff5ebbae69
bs4 is a dummy package managed by the developer of Beautiful Soup to prevent name squatting. The official name of PyPI’s Beautiful Soup Python package is beautifulsoup4. The bs4 package ensures that if you type pip install bs4 by mistake you will end up with Beautiful Soup. However, for requirements, it's cleaner to use the proper name. For downstream packaging in Fedora, this avoids the need of packaging the dummy package.
511 lines
18 KiB
Python
Executable file
511 lines
18 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# -*- coding: iso-8859-1 -*-
|
|
# Copyright (C) 2000-2014 Bastian Kleineidam
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
"""
|
|
Setup file for the distuils module.
|
|
|
|
It includes the following features:
|
|
- creation and installation of configuration files with installation data
|
|
- automatic detection and usage of GNU99 standard for C compiler
|
|
- automatic MANIFEST.in check
|
|
- automatic generation of .mo locale files
|
|
- automatic permission setting on POSIX systems for installed files
|
|
|
|
Because of all the features, this script is nasty and big.
|
|
Change it very carefully.
|
|
"""
|
|
from __future__ import print_function
|
|
import sys
|
|
if not (hasattr(sys, 'version_info') or
|
|
sys.version_info < (2, 7, 0, 'final', 0)):
|
|
raise SystemExit("This program requires Python 2.7 or later.")
|
|
import os
|
|
import re
|
|
import codecs
|
|
import subprocess
|
|
import stat
|
|
import glob
|
|
import shutil
|
|
try:
|
|
unicode
|
|
except NameError:
|
|
unicode = lambda x: x
|
|
|
|
# import Distutils stuff
|
|
from setuptools import setup
|
|
from distutils.core import Extension
|
|
from distutils.command.install_lib import install_lib
|
|
from distutils.command.build_ext import build_ext
|
|
from distutils.command.sdist import sdist
|
|
from distutils.command.clean import clean
|
|
from distutils.command.install_data import install_data
|
|
from distutils.dir_util import remove_tree
|
|
from distutils.file_util import write_file
|
|
from distutils import util, log
|
|
from distutils.core import Distribution
|
|
from distutils.command.build import build
|
|
|
|
# the application version
|
|
AppVersion = "9.4.0"
|
|
# the application name
|
|
AppName = "LinkChecker"
|
|
Description = "check links in web documents or full websites"
|
|
|
|
def get_long_description():
|
|
"""Try to read long description from README.rst."""
|
|
try:
|
|
with open('README.rst') as f:
|
|
return f.read()
|
|
except:
|
|
return Description
|
|
|
|
def normpath (path):
|
|
"""Norm a path name to platform specific notation."""
|
|
return os.path.normpath(path)
|
|
|
|
|
|
def cnormpath (path):
|
|
"""Norm a path name to platform specific notation and make it absolute."""
|
|
path = normpath(path)
|
|
if os.name == 'nt':
|
|
# replace slashes with backslashes
|
|
path = path.replace("/", "\\")
|
|
if not os.path.isabs(path):
|
|
path = normpath(os.path.join(sys.prefix, path))
|
|
return path
|
|
|
|
|
|
release_ro = re.compile(r"\(released (.+)\)")
|
|
def get_release_date ():
|
|
"""Parse and return relase date as string from doc/changelog.txt."""
|
|
fname = os.path.join("doc", "changelog.txt")
|
|
release_date = "unknown"
|
|
with open(fname) as fd:
|
|
# the release date is on the first line
|
|
line = fd.readline()
|
|
mo = release_ro.search(line)
|
|
if mo:
|
|
release_date = mo.groups(1)
|
|
return release_date
|
|
|
|
|
|
def get_portable():
|
|
"""Return portable flag as string."""
|
|
return os.environ.get('LINKCHECKER_PORTABLE', '0')
|
|
|
|
|
|
class MyInstallLib (install_lib, object):
|
|
"""Custom library installation."""
|
|
|
|
def install (self):
|
|
"""Install the generated config file."""
|
|
outs = super(MyInstallLib, self).install()
|
|
infile = self.create_conf_file()
|
|
outfile = os.path.join(self.install_dir, os.path.basename(infile))
|
|
self.copy_file(infile, outfile)
|
|
outs.append(outfile)
|
|
return outs
|
|
|
|
def create_conf_file (self):
|
|
"""Create configuration file."""
|
|
cmd_obj = self.distribution.get_command_obj("install")
|
|
cmd_obj.ensure_finalized()
|
|
# we have to write a configuration file because we need the
|
|
# <install_data> directory (and other stuff like author, url, ...)
|
|
# all paths are made absolute by cnormpath()
|
|
data = []
|
|
for d in ['purelib', 'platlib', 'lib', 'headers', 'scripts', 'data']:
|
|
attr = 'install_%s' % d
|
|
if cmd_obj.root:
|
|
# cut off root path prefix
|
|
cutoff = len(cmd_obj.root)
|
|
# don't strip the path separator
|
|
if cmd_obj.root.endswith(os.sep):
|
|
cutoff -= 1
|
|
val = getattr(cmd_obj, attr)[cutoff:]
|
|
else:
|
|
val = getattr(cmd_obj, attr)
|
|
if attr == 'install_data':
|
|
cdir = os.path.join(val, "share", "linkchecker")
|
|
data.append('config_dir = %r' % cnormpath(cdir))
|
|
elif attr == 'install_lib':
|
|
if cmd_obj.root:
|
|
_drive, tail = os.path.splitdrive(val)
|
|
if tail.startswith(os.sep):
|
|
tail = tail[1:]
|
|
self.install_lib = os.path.join(cmd_obj.root, tail)
|
|
else:
|
|
self.install_lib = val
|
|
data.append("%s = %r" % (attr, cnormpath(val)))
|
|
self.distribution.create_conf_file(data, directory=self.install_lib)
|
|
return self.get_conf_output()
|
|
|
|
def get_conf_output (self):
|
|
"""Get name of configuration file."""
|
|
return self.distribution.get_conf_filename(self.install_lib)
|
|
|
|
def get_outputs (self):
|
|
"""Add the generated config file to the list of outputs."""
|
|
outs = super(MyInstallLib, self).get_outputs()
|
|
conf_output = self.get_conf_output()
|
|
outs.append(conf_output)
|
|
if self.compile:
|
|
outs.extend(self._bytecode_filenames([conf_output]))
|
|
return outs
|
|
|
|
|
|
class MyInstallData (install_data, object):
|
|
"""Fix file permissions."""
|
|
|
|
def run (self):
|
|
"""Adjust permissions on POSIX systems."""
|
|
self.install_translations()
|
|
super(MyInstallData, self).run()
|
|
self.fix_permissions()
|
|
|
|
def install_translations (self):
|
|
"""Install compiled gettext catalogs."""
|
|
# A hack to fix https://github.com/linkchecker/linkchecker/issues/102
|
|
i18n_files = []
|
|
data_files = []
|
|
for dir, files in self.data_files:
|
|
if 'LC_MESSAGES' in dir:
|
|
i18n_files.append((dir, files))
|
|
else:
|
|
data_files.append((dir, files))
|
|
self.data_files = data_files
|
|
# We do almost the same thing that install_data.run() does, except
|
|
# we can assume everything in self.data_files is a (dir, files) tuple,
|
|
# and all files lists are non-empty. And for i18n files, instead of
|
|
# specifying the directory we instead specify the destination filename.
|
|
for dest, files in i18n_files:
|
|
dest = util.convert_path(dest)
|
|
if not os.path.isabs(dest):
|
|
dest = os.path.join(self.install_dir, dest)
|
|
elif self.root:
|
|
dest = util.change_root(self.root, dest)
|
|
self.mkpath(os.path.dirname(dest))
|
|
for data in files:
|
|
data = util.convert_path(data)
|
|
(out, _) = self.copy_file(data, dest)
|
|
self.outfiles.append(out)
|
|
|
|
def fix_permissions (self):
|
|
"""Set correct read permissions on POSIX systems. Might also
|
|
be possible by setting umask?"""
|
|
if os.name == 'posix' and not self.dry_run:
|
|
# Make the data files we just installed world-readable,
|
|
# and the directories world-executable as well.
|
|
for path in self.get_outputs():
|
|
mode = os.stat(path)[stat.ST_MODE]
|
|
if stat.S_ISDIR(mode):
|
|
mode |= 0o11
|
|
mode |= 0o44
|
|
os.chmod(path, mode)
|
|
|
|
|
|
class MyDistribution (Distribution, object):
|
|
"""Custom distribution class generating config file."""
|
|
|
|
def __init__ (self, attrs):
|
|
"""Set console and windows scripts."""
|
|
super(MyDistribution, self).__init__(attrs)
|
|
self.console = ['linkchecker']
|
|
|
|
def run_commands (self):
|
|
"""Generate config file and run commands."""
|
|
cwd = os.getcwd()
|
|
data = []
|
|
data.append('config_dir = %r' % os.path.join(cwd, "config"))
|
|
data.append("install_data = %r" % cwd)
|
|
data.append("install_scripts = %r" % cwd)
|
|
self.create_conf_file(data)
|
|
super(MyDistribution, self).run_commands()
|
|
|
|
def get_conf_filename (self, directory):
|
|
"""Get name for config file."""
|
|
return os.path.join(directory, "_%s_configdata.py" % self.get_name())
|
|
|
|
def create_conf_file (self, data, directory=None):
|
|
"""Create local config file from given data (list of lines) in
|
|
the directory (or current directory if not given)."""
|
|
data.insert(0, "# this file is automatically created by setup.py")
|
|
data.insert(0, "# -*- coding: iso-8859-1 -*-")
|
|
if directory is None:
|
|
directory = os.getcwd()
|
|
filename = self.get_conf_filename(directory)
|
|
# add metadata
|
|
metanames = ("name", "version", "author", "author_email",
|
|
"maintainer", "maintainer_email", "url",
|
|
"license", "description", "long_description",
|
|
"keywords", "platforms", "fullname", "contact",
|
|
"contact_email")
|
|
for name in metanames:
|
|
method = "get_" + name
|
|
val = getattr(self.metadata, method)()
|
|
if isinstance(val, str):
|
|
val = unicode(val)
|
|
cmd = "%s = %r" % (name, val)
|
|
data.append(cmd)
|
|
data.append('release_date = "%s"' % get_release_date())
|
|
data.append('portable = %s' % get_portable())
|
|
# write the config file
|
|
util.execute(write_file, (filename, data),
|
|
"creating %s" % filename, self.verbose >= 1, self.dry_run)
|
|
|
|
|
|
def cc_run (args):
|
|
"""Run the C compiler with a simple main program.
|
|
|
|
@return: successful exit flag
|
|
@rtype: bool
|
|
"""
|
|
prog = b"int main(){}\n"
|
|
pipe = subprocess.Popen(args,
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True)
|
|
pipe.communicate(input=prog)
|
|
if os.WIFEXITED(pipe.returncode):
|
|
return os.WEXITSTATUS(pipe.returncode) == 0
|
|
return False
|
|
|
|
|
|
def cc_supports_option (cc, option):
|
|
"""Check if the given C compiler supports the given option.
|
|
|
|
@return: True if the compiler supports the option, else False
|
|
@rtype: bool
|
|
"""
|
|
return cc_run([cc[0], "-E", option, "-"])
|
|
|
|
|
|
class MyBuildExt (build_ext, object):
|
|
"""Custom build extension command."""
|
|
|
|
def build_extensions (self):
|
|
"""Add -std=gnu99 to build options if supported."""
|
|
# For gcc >= 3 we can add -std=gnu99 to get rid of warnings.
|
|
extra = []
|
|
if self.compiler.compiler_type == 'unix':
|
|
option = "-std=gnu99"
|
|
if cc_supports_option(self.compiler.compiler, option):
|
|
extra.append(option)
|
|
# First, sanity-check the 'extensions' list
|
|
self.check_extensions_list(self.extensions)
|
|
for ext in self.extensions:
|
|
for opt in extra:
|
|
if opt not in ext.extra_compile_args:
|
|
ext.extra_compile_args.append(opt)
|
|
self.build_extension(ext)
|
|
|
|
|
|
def list_message_files (package, suffix=".mo"):
|
|
"""Return list of all found message files and their installation paths."""
|
|
for fname in glob.glob("po/*" + suffix):
|
|
# basename (without extension) is a locale name
|
|
localename = os.path.splitext(os.path.basename(fname))[0]
|
|
domainname = "%s.mo" % package.lower()
|
|
yield (fname, os.path.join(
|
|
"share", "locale", localename, "LC_MESSAGES", domainname))
|
|
|
|
|
|
def check_manifest ():
|
|
"""Snatched from roundup.sf.net.
|
|
Check that the files listed in the MANIFEST are present when the
|
|
source is unpacked."""
|
|
try:
|
|
f = open('MANIFEST')
|
|
except Exception:
|
|
print('\n*** SOURCE WARNING: The MANIFEST file is missing!')
|
|
return
|
|
try:
|
|
manifest = [l.strip() for l in f.readlines() if not l.startswith('#')]
|
|
finally:
|
|
f.close()
|
|
err = [line for line in manifest if not os.path.exists(line)]
|
|
if err:
|
|
n = len(manifest)
|
|
print('\n*** SOURCE WARNING: There are files missing (%d/%d found)!' %
|
|
(n - len(err), n))
|
|
print('\nMissing: '.join(err))
|
|
|
|
|
|
class MyBuild (build, object):
|
|
"""Custom build command."""
|
|
|
|
def run (self):
|
|
"""Check MANIFEST before building."""
|
|
check_manifest()
|
|
build.run(self)
|
|
|
|
|
|
class MyClean (clean, object):
|
|
"""Custom clean command."""
|
|
|
|
def run (self):
|
|
"""Remove share directory on clean."""
|
|
if self.all:
|
|
# remove share directory
|
|
directory = os.path.join("build", "share")
|
|
if os.path.exists(directory):
|
|
remove_tree(directory, dry_run=self.dry_run)
|
|
else:
|
|
log.warn("'%s' does not exist -- can't clean it", directory)
|
|
clean.run(self)
|
|
|
|
|
|
class MySdist (sdist, object):
|
|
"""Custom sdist command."""
|
|
|
|
def get_file_list (self):
|
|
"""Add MANIFEST to the file list."""
|
|
super(MySdist, self).get_file_list()
|
|
self.filelist.append("MANIFEST")
|
|
|
|
|
|
# global include dirs
|
|
include_dirs = []
|
|
# global macros
|
|
define_macros = []
|
|
# compiler args
|
|
extra_compile_args = []
|
|
# library directories
|
|
library_dirs = []
|
|
# libraries
|
|
libraries = []
|
|
# scripts
|
|
scripts = ['linkchecker']
|
|
|
|
if os.name == 'nt':
|
|
# windows does not have unistd.h
|
|
define_macros.append(('YY_NO_UNISTD_H', None))
|
|
else:
|
|
extra_compile_args.append("-pedantic")
|
|
|
|
if sys.platform == 'darwin':
|
|
define_macros.extend([('HAVE_STRLCPY', None), ('HAVE_STRLCAT', None)])
|
|
|
|
myname = "Bastian Kleineidam"
|
|
myemail = "bastian.kleineidam@web.de"
|
|
|
|
data_files = [
|
|
('share/linkchecker',
|
|
['config/linkcheckerrc']),
|
|
('share/linkchecker/examples',
|
|
['cgi-bin/lconline/leer.html.en',
|
|
'cgi-bin/lconline/leer.html.de',
|
|
'cgi-bin/lconline/index.html',
|
|
'cgi-bin/lconline/lc_cgi.html.en',
|
|
'cgi-bin/lconline/lc_cgi.html.de',
|
|
'cgi-bin/lconline/check.js',
|
|
'cgi-bin/lc.wsgi',
|
|
'config/linkchecker.apache2.conf',
|
|
]),
|
|
]
|
|
|
|
for (src, dst) in list_message_files(AppName):
|
|
data_files.append((dst, [src]))
|
|
|
|
if os.name == 'posix':
|
|
data_files.append(('share/man/man1', ['doc/en/linkchecker.1']))
|
|
data_files.append(('share/man/man5', ['doc/en/linkcheckerrc.5']))
|
|
data_files.append(('share/man/de/man1', ['doc/de/linkchecker.1']))
|
|
data_files.append(('share/man/de/man5', ['doc/de/linkcheckerrc.5']))
|
|
data_files.append(('share/linkchecker/examples',
|
|
['config/linkchecker-completion',
|
|
'doc/examples/check_blacklist.sh',
|
|
'doc/examples/check_for_x_errors.sh',
|
|
'doc/examples/check_urls.sh']))
|
|
data_files.append(('share/applications', ['doc/linkchecker.desktop']))
|
|
|
|
args = dict(
|
|
name = AppName,
|
|
version = AppVersion,
|
|
description = Description,
|
|
keywords = "link,url,site,checking,crawling,verification,validation",
|
|
author = myname,
|
|
author_email = myemail,
|
|
maintainer = myname,
|
|
maintainer_email = myemail,
|
|
url = "https://linkchecker.github.io/linkchecker/",
|
|
license = "GPL",
|
|
long_description = get_long_description(),
|
|
distclass = MyDistribution,
|
|
cmdclass = {
|
|
'install_lib': MyInstallLib,
|
|
'install_data': MyInstallData,
|
|
'build_ext': MyBuildExt,
|
|
'build': MyBuild,
|
|
'clean': MyClean,
|
|
'sdist': MySdist,
|
|
},
|
|
packages = [
|
|
'linkcheck',
|
|
'linkcheck.bookmarks',
|
|
'linkcheck.cache',
|
|
'linkcheck.checker',
|
|
'linkcheck.configuration',
|
|
'linkcheck.director',
|
|
'linkcheck.htmlutil',
|
|
'linkcheck.HtmlParser',
|
|
'linkcheck.logger',
|
|
'linkcheck.network',
|
|
'linkcheck.parser',
|
|
'linkcheck.plugins',
|
|
],
|
|
ext_modules = [
|
|
Extension('linkcheck.HtmlParser.htmlsax',
|
|
sources = [
|
|
'linkcheck/HtmlParser/htmllex.c',
|
|
'linkcheck/HtmlParser/htmlparse.c',
|
|
'linkcheck/HtmlParser/s_util.c',
|
|
],
|
|
extra_compile_args = extra_compile_args,
|
|
library_dirs = library_dirs,
|
|
libraries = libraries,
|
|
define_macros = define_macros + [('YY_NO_INPUT', None)],
|
|
include_dirs = include_dirs + [normpath("linkcheck/HtmlParser")],
|
|
),
|
|
],
|
|
scripts = scripts,
|
|
data_files = data_files,
|
|
classifiers = [
|
|
'Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking',
|
|
'Development Status :: 5 - Production/Stable',
|
|
'License :: OSI Approved :: GNU General Public License (GPL)',
|
|
'Programming Language :: Python',
|
|
'Programming Language :: C',
|
|
],
|
|
options = {
|
|
},
|
|
# Requirements, usable with setuptools or the new Python packaging module.
|
|
install_requires = [
|
|
'requests >= 2.4',
|
|
'dnspython',
|
|
'beautifulsoup4',
|
|
'pyxdg',
|
|
'future',
|
|
],
|
|
# Commented out since they are untested and not officially supported.
|
|
# See also doc/install.txt for more detailed dependency documentation.
|
|
#extra_requires = {
|
|
# "IP country info": ['GeoIP'], # http://www.maxmind.com/app/python
|
|
# "GNOME proxies": ['pygtk'], # http://www.pygtk.org/downloads.html
|
|
# "Bash completion": ['argcomplete'], # https://pypi.python.org/pypi/argcomplete
|
|
# "Memory debugging": ['meliae'], # https://launchpad.net/meliae
|
|
#}
|
|
)
|
|
setup(**args)
|