From fc9efceb46be7f9e67bc2a92162c70c68cbc81a7 Mon Sep 17 00:00:00 2001 From: Chris Mayo Date: Thu, 28 May 2020 19:51:38 +0100 Subject: [PATCH] Run black on linkchecker and setup.py --- linkchecker | 531 +++++++++++++++++++++++++++++++++++----------------- setup.py | 188 +++++++++++-------- 2 files changed, 467 insertions(+), 252 deletions(-) diff --git a/linkchecker b/linkchecker index 12a2d02f..26f3d349 100755 --- a/linkchecker +++ b/linkchecker @@ -25,15 +25,22 @@ import os import pprint import argparse import getpass + # installs _() and _n() gettext functions into global namespace import linkcheck from linkcheck import logconf, LOG_CMDLINE + logconf.init_log_config() # override argparse gettext method with the one from linkcheck.init_i18n() -#argparse._ = _ +# argparse._ = _ # now import the rest of the linkchecker gang -from linkcheck.cmdline import print_version, print_usage, aggregate_url, \ - LCArgumentParser, print_plugins +from linkcheck.cmdline import ( + print_version, + print_usage, + aggregate_url, + LCArgumentParser, + print_plugins, +) from linkcheck import log, i18n, strformat import linkcheck.checker import linkcheck.configuration @@ -41,6 +48,7 @@ import linkcheck.fileutil import linkcheck.logger import linkcheck.ansicolor from linkcheck.director import console, check_urls, get_aggregate + # optional modules has_argcomplete = linkcheck.fileutil.has_module("argcomplete") has_profile = linkcheck.fileutil.has_module("yappi") @@ -52,7 +60,8 @@ _username = None _password = None # usage texts -Notes = _("""NOTES +Notes = _( + """NOTES o URLs on the command line starting with "ftp." are treated like "ftp://ftp.", URLs starting with "www." are treated like "http://www.". You can also give local files as arguments. @@ -66,9 +75,11 @@ Notes = _("""NOTES o You can supply multiple user/password pairs in a configuration file. o When checking 'news:' links the given NNTP host doesn't need to be the same as the host of the user browsing your pages. -""") +""" +) -ProxySupport = _("""PROXY SUPPORT +ProxySupport = _( + """PROXY SUPPORT To use a proxy on Unix or Windows set $http_proxy, $https_proxy or $ftp_proxy to the proxy URL. The URL should be of the form "http://[:@][:]". @@ -91,18 +102,22 @@ Setting a proxy on the Windows command prompt: set http_proxy=http://proxy.example.com:8080 -""") +""" +) -RegularExpressions = _("""REGULAR EXPRESSIONS +RegularExpressions = _( + """REGULAR EXPRESSIONS Only Python regular expressions are accepted by LinkChecker. See http://www.amk.ca/python/howto/regex/ for an introduction in regular expressions. The only addition is that a leading exclamation mark negates the regular expression. -""") +""" +) -CookieFormat = _("""COOKIE FILES +CookieFormat = _( + """COOKIE FILES A cookie file contains standard RFC 805 header data with the following possible names: Scheme (optional) @@ -128,16 +143,20 @@ Set-cookie: spam="egg" Scheme: https Host: example.com Set-cookie: baggage="elitist"; comment="hologram" -""") +""" +) -Retval = _(r"""RETURN VALUE +Retval = _( + r"""RETURN VALUE The return value is non-zero when o invalid links were found or o warnings were found warnings are enabled o a program error occurred -""") +""" +) -Examples = _(r"""EXAMPLES +Examples = _( + r"""EXAMPLES The most common use checks the given domain recursively, plus any single URL pointing outside of the domain: linkchecker http://www.example.org/ @@ -159,9 +178,11 @@ You can skip the "http://" url part if the domain starts with "www.": You can skip the "ftp://" url part if the domain starts with "ftp.": linkchecker -r0 ftp.example.org -""") +""" +) -LoggerTypes = _(r"""OUTPUT TYPES +LoggerTypes = _( + r"""OUTPUT TYPES Note that by default only errors and warnings are logged. You should use the --verbose option to see valid URLs, and when outputting a sitemap graph format. @@ -184,16 +205,36 @@ blacklist ~/.linkchecker/blacklist which only contains entries with invalid URLs and the number of times they have failed. none Logs nothing. Suitable for debugging or checking the exit code. -""") +""" +) -Warnings = _(r"""IGNORE WARNINGS +Warnings = ( + _( + r"""IGNORE WARNINGS The following warnings are recognized in the 'ignorewarnings' config file entry: -""") + \ -"\n".join([" o %s - %s" % (tag, desc) \ - for tag, desc in sorted(linkcheck.checker.const.Warnings.items())]) +""" + ) + + "\n".join( + [ + " o %s - %s" % (tag, desc) + for tag, desc in sorted(linkcheck.checker.const.Warnings.items()) + ] + ) +) -Epilog = "\n".join((Examples, LoggerTypes, RegularExpressions, CookieFormat, ProxySupport, Notes, Retval, Warnings)) +Epilog = "\n".join( + ( + Examples, + LoggerTypes, + RegularExpressions, + CookieFormat, + ProxySupport, + Notes, + Retval, + Warnings, + ) +) def has_encoding(encoding): @@ -204,10 +245,10 @@ def has_encoding(encoding): except LookupError: return False + # instantiate option parser and configure options argparser = LCArgumentParser( - epilog=Epilog, - formatter_class=argparse.RawDescriptionHelpFormatter + epilog=Epilog, formatter_class=argparse.RawDescriptionHelpFormatter ) @@ -217,40 +258,68 @@ config.set_status_logger(console.StatusLogger()) ################# general options ################## group = argparser.add_argument_group(_("General options")) -group.add_argument("-f", "--config", dest="configfile", - metavar="FILENAME", - help=_( -"""Use FILENAME as configuration file. Per default LinkChecker uses +group.add_argument( + "-f", + "--config", + dest="configfile", + metavar="FILENAME", + help=_( + """Use FILENAME as configuration file. Per default LinkChecker uses ~/.linkchecker/linkcheckerrc (under Windows -%%HOMEPATH%%\\.linkchecker\\linkcheckerrc).""")) -group.add_argument("-t", "--threads", type=int, metavar="NUMBER", - help=_( -"""Generate no more than the given number of threads. Default number -of threads is 10. To disable threading specify a non-positive number.""")) -group.add_argument("-V", "--version", action="store_true", - help=_("""Print version and exit.""")) -group.add_argument("--list-plugins", action="store_true", dest="listplugins", - help=_( -"""Print available check plugins and exit.""")) -group.add_argument("--stdin", action="store_true", - help=_( -"""Read list of white-space separated URLs to check from stdin.""")) +%%HOMEPATH%%\\.linkchecker\\linkcheckerrc).""" + ), +) +group.add_argument( + "-t", + "--threads", + type=int, + metavar="NUMBER", + help=_( + """Generate no more than the given number of threads. Default number +of threads is 10. To disable threading specify a non-positive number.""" + ), +) +group.add_argument( + "-V", "--version", action="store_true", help=_("""Print version and exit.""") +) +group.add_argument( + "--list-plugins", + action="store_true", + dest="listplugins", + help=_("""Print available check plugins and exit."""), +) +group.add_argument( + "--stdin", + action="store_true", + help=_("""Read list of white-space separated URLs to check from stdin."""), +) ################# output options ################## group = argparser.add_argument_group(_("Output options")) -group.add_argument("-D", "--debug", action="append", metavar="STRING", - help=_("""Print debugging output for the given logger. +group.add_argument( + "-D", + "--debug", + action="append", + metavar="STRING", + help=_( + """Print debugging output for the given logger. Available loggers are %(lognamelist)s. Specifying 'all' is an alias for specifying all available loggers. The option can be given multiple times to debug with more than one logger. -For accurate results, threading will be disabled during debug runs.""") % \ -{"lognamelist": logconf.lognamelist}) -group.add_argument("-F", "--file-output", action="append", - dest="fileoutput", metavar="TYPE[/ENCODING[/FILENAME]]", - help=_( -"""Output to a file linkchecker-out.TYPE, $HOME/.linkchecker/blacklist for +For accurate results, threading will be disabled during debug runs.""" + ) + % {"lognamelist": logconf.lognamelist}, +) +group.add_argument( + "-F", + "--file-output", + action="append", + dest="fileoutput", + metavar="TYPE[/ENCODING[/FILENAME]]", + help=_( + """Output to a file linkchecker-out.TYPE, $HOME/.linkchecker/blacklist for 'blacklist' output, or FILENAME if specified. The ENCODING specifies the output encoding, the default is that of your locale. @@ -260,88 +329,176 @@ else if the file already exists, it will be overwritten. You can specify this option more than once. Valid file output types are %(loggertypes)s. You can specify this option multiple times to output to more than one file. Default is no file output. Note that you can -suppress all console output with the option '-o none'.""") % \ -{'loggertypes': linkcheck.logger.LoggerKeys}) -group.add_argument("--no-status", action="store_false", dest="status", - default=True, help=_( -"""Do not print check status messages.""")) -group.add_argument("--no-warnings", action="store_false", dest="warnings", - help=_("""Don't log warnings. Default is to log warnings.""")) -group.add_argument("-o", "--output", dest="output", metavar="TYPE[/ENCODING]", - help=_( -"""Specify output as %(loggertypes)s. Default output type is text. +suppress all console output with the option '-o none'.""" + ) + % {"loggertypes": linkcheck.logger.LoggerKeys}, +) +group.add_argument( + "--no-status", + action="store_false", + dest="status", + default=True, + help=_("""Do not print check status messages."""), +) +group.add_argument( + "--no-warnings", + action="store_false", + dest="warnings", + help=_("""Don't log warnings. Default is to log warnings."""), +) +group.add_argument( + "-o", + "--output", + dest="output", + metavar="TYPE[/ENCODING]", + help=_( + """Specify output as %(loggertypes)s. Default output type is text. The ENCODING specifies the output encoding, the default is that of your locale. -Valid encodings are listed at """ \ -"""http://docs.python.org/lib/standard-encodings.html.""") % \ -{'loggertypes': linkcheck.logger.LoggerKeys}) -group.add_argument("--profile", action="store_true", dest="profile", - help=argparse.SUPPRESS) -group.add_argument("-q", "--quiet", action="store_true", dest="quiet", - help=_( -"""Quiet operation, an alias for '-o none'. -This is only useful with -F.""")) -group.add_argument("--trace", action="store_true", dest="trace", - help=argparse.SUPPRESS) -group.add_argument("-v", "--verbose", action="store_true", dest="verbose", - help=_( -"""Log all URLs. Default is to log only errors and warnings.""")) +Valid encodings are listed at """ + """http://docs.python.org/lib/standard-encodings.html.""" + ) + % {"loggertypes": linkcheck.logger.LoggerKeys}, +) +group.add_argument( + "--profile", action="store_true", dest="profile", help=argparse.SUPPRESS +) +group.add_argument( + "-q", + "--quiet", + action="store_true", + dest="quiet", + help=_( + """Quiet operation, an alias for '-o none'. +This is only useful with -F.""" + ), +) +group.add_argument("--trace", action="store_true", dest="trace", help=argparse.SUPPRESS) +group.add_argument( + "-v", + "--verbose", + action="store_true", + dest="verbose", + help=_("""Log all URLs. Default is to log only errors and warnings."""), +) ################# checking options ################## group = argparser.add_argument_group(_("Checking options")) -group.add_argument("--cookiefile", dest="cookiefile", metavar="FILENAME", - help=_( -"""Read a file with initial cookie data. The cookie data format is -explained below.""")) +group.add_argument( + "--cookiefile", + dest="cookiefile", + metavar="FILENAME", + help=_( + """Read a file with initial cookie data. The cookie data format is +explained below.""" + ), +) # const because store_false doesn't detect absent flags -group.add_argument("--no-robots", action="store_const", const=False, - dest="norobotstxt", help=_("Disable robots.txt checks")) -group.add_argument("--check-extern", action="store_true", - dest="checkextern", help=_("""Check external URLs.""")) -group.add_argument("--ignore-url", action="append", metavar="REGEX", - dest="externstrict", help=_( -"""Only check syntax of URLs matching the given regular expression. - This option can be given multiple times.""")) -group.add_argument("--no-follow-url", action="append", metavar="REGEX", - dest="extern", help=_( -"""Check but do not recurse into URLs matching the given regular -expression. This option can be given multiple times.""")) -group.add_argument("-N", "--nntp-server", dest="nntpserver", metavar="STRING", - help=_( -"""Specify an NNTP server for 'news:...' links. Default is the +group.add_argument( + "--no-robots", + action="store_const", + const=False, + dest="norobotstxt", + help=_("Disable robots.txt checks"), +) +group.add_argument( + "--check-extern", + action="store_true", + dest="checkextern", + help=_("""Check external URLs."""), +) +group.add_argument( + "--ignore-url", + action="append", + metavar="REGEX", + dest="externstrict", + help=_( + """Only check syntax of URLs matching the given regular expression. + This option can be given multiple times.""" + ), +) +group.add_argument( + "--no-follow-url", + action="append", + metavar="REGEX", + dest="extern", + help=_( + """Check but do not recurse into URLs matching the given regular +expression. This option can be given multiple times.""" + ), +) +group.add_argument( + "-N", + "--nntp-server", + dest="nntpserver", + metavar="STRING", + help=_( + """Specify an NNTP server for 'news:...' links. Default is the environment variable NNTP_SERVER. If no host is given, -only the syntax of the link is checked.""")) -group.add_argument("-p", "--password", action="store_false", dest="password", - default=False, - help=_( -"""Read a password from console and use it for HTTP and FTP authorization. +only the syntax of the link is checked.""" + ), +) +group.add_argument( + "-p", + "--password", + action="store_false", + dest="password", + default=False, + help=_( + """Read a password from console and use it for HTTP and FTP authorization. For FTP the default password is 'anonymous@'. For HTTP there is -no default password. See also -u.""")) -group.add_argument("-r", "--recursion-level", type=int, - dest="recursionlevel", metavar="NUMBER", - help=_( -"""Check recursively all links up to given depth. A negative depth -will enable infinite recursion. Default depth is infinite.""")) -group.add_argument("--timeout", type=int, dest="timeout", - metavar="NUMBER", - help=_( -"""Set the timeout for connection attempts in seconds. The default -timeout is %d seconds.""") % config["timeout"]) -group.add_argument("-u", "--user", dest="username", metavar="STRING", - help=_( -"""Try the given username for HTTP and FTP authorization. +no default password. See also -u.""" + ), +) +group.add_argument( + "-r", + "--recursion-level", + type=int, + dest="recursionlevel", + metavar="NUMBER", + help=_( + """Check recursively all links up to given depth. A negative depth +will enable infinite recursion. Default depth is infinite.""" + ), +) +group.add_argument( + "--timeout", + type=int, + dest="timeout", + metavar="NUMBER", + help=_( + """Set the timeout for connection attempts in seconds. The default +timeout is %d seconds.""" + ) + % config["timeout"], +) +group.add_argument( + "-u", + "--user", + dest="username", + metavar="STRING", + help=_( + """Try the given username for HTTP and FTP authorization. For FTP the default username is 'anonymous'. For HTTP there is -no default username. See also -p.""")) -group.add_argument("--user-agent", dest="useragent", metavar="STRING", - help=_( -"""Specify the User-Agent string to send to the HTTP server, for example +no default username. See also -p.""" + ), +) +group.add_argument( + "--user-agent", + dest="useragent", + metavar="STRING", + help=_( + """Specify the User-Agent string to send to the HTTP server, for example "Mozilla/4.0". The default is "LinkChecker/X.Y" where X.Y is the current -version of LinkChecker.""")) +version of LinkChecker.""" + ), +) -argparser.add_argument('url', nargs='*') +argparser.add_argument("url", nargs="*") ################# auto completion ##################### if has_argcomplete: import argcomplete + argcomplete.autocomplete(argparser) @@ -368,10 +525,13 @@ if options.debug: allowed_debugs = logconf.lognames.keys() for _name in options.debug: if _name not in allowed_debugs: - print_usage(_("Invalid debug level %(level)r") % {'level': _name}) + print_usage(_("Invalid debug level %(level)r") % {"level": _name}) logconf.set_debug(options.debug) -log.debug(LOG_CMDLINE, _("Python %(version)s on %(platform)s") % \ - {"version": sys.version, "platform": sys.platform}) +log.debug( + LOG_CMDLINE, + _("Python %(version)s on %(platform)s") + % {"version": sys.version, "platform": sys.platform}, +) # read configuration files try: files = [] @@ -380,8 +540,7 @@ try: if os.path.isfile(path): files.append(path) else: - log.warn(LOG_CMDLINE, - _("Unreadable config file: %r"), options.configfile) + log.warn(LOG_CMDLINE, _("Unreadable config file: %r"), options.configfile) config.read(files=files) except linkcheck.LinkCheckerError as msg: # config error @@ -398,18 +557,20 @@ if options.version: if not options.warnings: config["warnings"] = options.warnings if options.externstrict: - pats = [linkcheck.get_link_pat(arg, strict=True) \ - for arg in options.externstrict] + pats = [linkcheck.get_link_pat(arg, strict=True) for arg in options.externstrict] config["externlinks"].extend(pats) if options.extern: pats = [linkcheck.get_link_pat(arg) for arg in options.extern] config["externlinks"].extend(pats) if options.norobotstxt is not None: - config['robotstxt'] = options.norobotstxt + config["robotstxt"] = options.norobotstxt if options.checkextern: config["checkextern"] = True elif not config["checkextern"]: - log.info(LOG_CMDLINE, "Checking intern URLs only; use --check-extern to check extern URLs.") + log.info( + LOG_CMDLINE, + "Checking intern URLs only; use --check-extern to check extern URLs.", + ) if options.output: if "/" in options.output: @@ -419,49 +580,61 @@ if options.output: logtype = logtype.lower() if logtype not in linkcheck.logger.LoggerNames: print_usage( - _("Unknown logger type %(type)r in %(output)r for option %(option)s") % \ - {"type": logtype, "output": options.output, "option": "'-o, --output'"}) - if logtype != 'none' and not has_encoding(encoding): + _("Unknown logger type %(type)r in %(output)r for option %(option)s") + % {"type": logtype, "output": options.output, "option": "'-o, --output'"} + ) + if logtype != "none" and not has_encoding(encoding): print_usage( - _("Unknown encoding %(encoding)r in %(output)r for option %(option)s") % \ - {"encoding": encoding, "output": options.output, - "option": "'-o, --output'"}) - config['output'] = logtype - config['logger'] = config.logger_new(logtype, encoding=encoding) + _("Unknown encoding %(encoding)r in %(output)r for option %(option)s") + % { + "encoding": encoding, + "output": options.output, + "option": "'-o, --output'", + } + ) + config["output"] = logtype + config["logger"] = config.logger_new(logtype, encoding=encoding) if options.fileoutput: - ns = {'fileoutput': 1} + ns = {"fileoutput": 1} for arg in options.fileoutput: ftype = arg # look for (optional) filename and encoding - if '/' in ftype: - ftype, suffix = ftype.split('/', 1) + if "/" in ftype: + ftype, suffix = ftype.split("/", 1) if suffix: if has_encoding(suffix): # it was an encoding - ns['encoding'] = suffix - elif '/' in suffix: + ns["encoding"] = suffix + elif "/" in suffix: # look for (optional) encoding - encoding, filename = suffix.split('/', 1) + encoding, filename = suffix.split("/", 1) if has_encoding(encoding): - ns['encoding'] = encoding - ns['filename'] = filename + ns["encoding"] = encoding + ns["filename"] = filename else: - ns['filename'] = suffix + ns["filename"] = suffix else: - ns['filename'] = suffix + ns["filename"] = suffix if ftype not in linkcheck.logger.LoggerNames: print_usage( - _("Unknown logger type %(type)r in %(output)r for option %(option)s") % \ - {"type": ftype, "output": options.fileoutput, - "option": "'-F, --file-output'"}) - if ftype != 'none' and 'encoding' in ns and \ - not has_encoding(ns['encoding']): + _("Unknown logger type %(type)r in %(output)r for option %(option)s") + % { + "type": ftype, + "output": options.fileoutput, + "option": "'-F, --file-output'", + } + ) + if ftype != "none" and "encoding" in ns and not has_encoding(ns["encoding"]): print_usage( - _("Unknown encoding %(encoding)r in %(output)r for option %(option)s") % \ - {"encoding": ns['encoding'], "output": options.fileoutput, - "option": "'-F, --file-output'"}) + _("Unknown encoding %(encoding)r in %(output)r for option %(option)s") + % { + "encoding": ns["encoding"], + "output": options.fileoutput, + "option": "'-F, --file-output'", + } + ) logger = config.logger_new(ftype, **ns) - config['fileoutput'].append(logger) + config["fileoutput"].append(logger) if options.nntpserver: config["nntpserver"] = options.nntpserver if options.username: @@ -469,8 +642,9 @@ if options.username: constructauth = True if options.password: if _username: - msg = _("Enter LinkChecker HTTP/FTP password for user %(user)s:") % \ - {"user": _username} + msg = _("Enter LinkChecker HTTP/FTP password for user %(user)s:") % { + "user": _username + } else: msg = _("Enter LinkChecker HTTP/FTP password:") _password = getpass.getpass(console.encode(msg)) @@ -478,11 +652,11 @@ if options.password: if options.profile: do_profile = options.profile if options.quiet: - config['logger'] = config.logger_new('none') + config["logger"] = config.logger_new("none") if options.recursionlevel is not None: config["recursionlevel"] = options.recursionlevel if options.status: - config['status'] = options.status + config["status"] = options.status if options.threads is not None: if options.threads < 1: options.threads = 0 @@ -491,8 +665,10 @@ if options.timeout is not None: if options.timeout > 0: config["timeout"] = options.timeout else: - print_usage(_("Illegal argument %(arg)r for option %(option)s") % \ - {"arg": options.timeout, "option": "'--timeout'"}) + print_usage( + _("Illegal argument %(arg)r for option %(option)s") + % {"arg": options.timeout, "option": "'--timeout'"} + ) if options.listplugins: print_plugins(config["pluginfolders"]) if options.verbose: @@ -500,7 +676,7 @@ if options.verbose: config["verbose"] = True config["warnings"] = True if options.cookiefile is not None: - config['cookiefile'] = options.cookiefile + config["cookiefile"] = options.cookiefile if constructauth: config.add_auth(pattern=".+", user=_username, password=_password) # read missing passwords @@ -508,8 +684,10 @@ for entry in config["authentication"]: if entry["password"] is None: attrs = entry.copy() attrs["strpattern"] = attrs["pattern"].pattern - msg = _("Enter LinkChecker password for user %(user)s" \ - " at %(strpattern)s:") % attrs + msg = ( + _("Enter LinkChecker password for user %(user)s" " at %(strpattern)s:") + % attrs + ) entry["password"] = getpass.getpass(msg) if options.useragent is not None: config["useragent"] = options.useragent @@ -522,8 +700,7 @@ if options.cookiefile is not None: # now sanitize the configuration config.sanitize() -log.debug(LOG_CMDLINE, "configuration: %s", - pprint.pformat(sorted(config.items()))) +log.debug(LOG_CMDLINE, "configuration: %s", pprint.pformat(sorted(config.items()))) # prepare checking queue aggregate = get_aggregate(config) @@ -532,6 +709,7 @@ if options.trace: config["trace"] = True # start trace in mainthread import linkcheck.trace + linkcheck.trace.trace_filter([r"^linkcheck"]) linkcheck.trace.trace_on() # add urls to queue @@ -547,22 +725,32 @@ else: if do_profile: if has_profile: if os.path.exists(_profile): - print(_("""Overwrite profiling file %(file)r? -Press Ctrl-C to cancel, RETURN to continue.""") % {"file": _profile}) + print( + _( + """Overwrite profiling file %(file)r? +Press Ctrl-C to cancel, RETURN to continue.""" + ) + % {"file": _profile} + ) try: input() except KeyboardInterrupt: print("", _("Canceled."), file=sys.stderr, sep="\n") sys.exit(1) else: - log.warn(LOG_CMDLINE, - _("The `yappi' Python module is not installed," - " therefore the --profile option is disabled.")) + log.warn( + LOG_CMDLINE, + _( + "The `yappi' Python module is not installed," + " therefore the --profile option is disabled." + ), + ) do_profile = False # finally, start checking if do_profile: import yappi + yappi.start() check_urls(aggregate) yappi.stop() @@ -571,6 +759,7 @@ else: check_urls(aggregate) if config["debugmemory"]: import linkcheck.memoryutil + if has_meliae: log.info(LOG_CMDLINE, _("Dumping memory statistics...")) filename = linkcheck.memoryutil.write_memory_dump() @@ -579,10 +768,10 @@ if config["debugmemory"]: else: log.warn(LOG_CMDLINE, linkcheck.memoryutil.MemoryDebugMsg) -stats = config['logger'].stats +stats = config["logger"].stats # on internal errors, exit with status 2 if stats.internal_errors: sys.exit(2) # on errors or printed warnings, exit with status 1 -if stats.errors or (stats.warnings_printed and config['warnings']): +if stats.errors or (stats.warnings_printed and config["warnings"]): sys.exit(1) diff --git a/setup.py b/setup.py index 569e1a4d..a3f1ce9f 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,8 @@ Because of all the features, this script is nasty and big. Change it very carefully. """ import sys -if sys.version_info < (3, 5, 0, 'final', 0): + +if sys.version_info < (3, 5, 0, "final", 0): raise SystemExit("This program requires Python 3.5 or later.") import os import re @@ -51,14 +52,16 @@ AppVersion = "10.0.0.dev0" AppName = "LinkChecker" Description = "check links in web documents or full websites" + def get_long_description(): """Try to read long description from README.rst.""" try: - with open('README.rst') as f: + with open("README.rst") as f: return f.read() except: return Description + def normpath(path): """Norm a path name to platform specific notation.""" return os.path.normpath(path) @@ -67,7 +70,7 @@ def normpath(path): def cnormpath(path): """Norm a path name to platform specific notation and make it absolute.""" path = normpath(path) - if os.name == 'nt': + if os.name == "nt": # replace slashes with backslashes path = path.replace("/", "\\") if not os.path.isabs(path): @@ -76,6 +79,8 @@ def cnormpath(path): release_ro = re.compile(r"\(released (.+)\)") + + def get_release_date(): """Parse and return relase date as string from doc/changelog.txt.""" fname = os.path.join("doc", "changelog.txt") @@ -91,7 +96,7 @@ def get_release_date(): def get_portable(): """Return portable flag as string.""" - return os.environ.get('LINKCHECKER_PORTABLE', '0') + return os.environ.get("LINKCHECKER_PORTABLE", "0") class MyInstallLib(install_lib): @@ -114,8 +119,8 @@ class MyInstallLib(install_lib): # directory (and other stuff like author, url, ...) # all paths are made absolute by cnormpath() data = [] - for d in ['purelib', 'platlib', 'lib', 'headers', 'scripts', 'data']: - attr = 'install_%s' % d + for d in ["purelib", "platlib", "lib", "headers", "scripts", "data"]: + attr = "install_%s" % d if cmd_obj.root: # cut off root path prefix cutoff = len(cmd_obj.root) @@ -125,10 +130,10 @@ class MyInstallLib(install_lib): val = getattr(cmd_obj, attr)[cutoff:] else: val = getattr(cmd_obj, attr) - if attr == 'install_data': + if attr == "install_data": cdir = os.path.join(val, "share", "linkchecker") - data.append('config_dir = %r' % cnormpath(cdir)) - elif attr == 'install_lib': + data.append("config_dir = %r" % cnormpath(cdir)) + elif attr == "install_lib": if cmd_obj.root: _drive, tail = os.path.splitdrive(val) if tail.startswith(os.sep): @@ -169,7 +174,7 @@ class MyInstallData(install_data): i18n_files = [] data_files = [] for dir, files in self.data_files: - if 'LC_MESSAGES' in dir: + if "LC_MESSAGES" in dir: i18n_files.append((dir, files)) else: data_files.append((dir, files)) @@ -193,7 +198,7 @@ class MyInstallData(install_data): def fix_permissions(self): """Set correct read permissions on POSIX systems. Might also be possible by setting umask?""" - if os.name == 'posix' and not self.dry_run: + if os.name == "posix" and not self.dry_run: # Make the data files we just installed world-readable, # and the directories world-executable as well. for path in self.get_outputs(): @@ -210,13 +215,13 @@ class MyDistribution(Distribution): def __init__(self, attrs): """Set console and windows scripts.""" super(MyDistribution, self).__init__(attrs) - self.console = ['linkchecker'] + self.console = ["linkchecker"] def run_commands(self): """Generate config file and run commands.""" cwd = os.getcwd() data = [] - data.append('config_dir = %r' % os.path.join(cwd, "config")) + data.append("config_dir = %r" % os.path.join(cwd, "config")) data.append("install_data = %r" % cwd) data.append("install_scripts = %r" % cwd) self.create_conf_file(data) @@ -235,21 +240,38 @@ class MyDistribution(Distribution): directory = os.getcwd() filename = self.get_conf_filename(directory) # add metadata - metanames = ("name", "version", "author", "author_email", - "maintainer", "maintainer_email", "url", - "license", "description", "long_description", - "keywords", "platforms", "fullname", "contact", - "contact_email") + metanames = ( + "name", + "version", + "author", + "author_email", + "maintainer", + "maintainer_email", + "url", + "license", + "description", + "long_description", + "keywords", + "platforms", + "fullname", + "contact", + "contact_email", + ) for name in metanames: method = "get_" + name val = getattr(self.metadata, method)() cmd = "%s = %r" % (name, val) data.append(cmd) data.append('release_date = "%s"' % get_release_date()) - data.append('portable = %s' % get_portable()) + data.append("portable = %s" % get_portable()) # write the config file - util.execute(write_file, (filename, data), - "creating %s" % filename, self.verbose >= 1, self.dry_run) + util.execute( + write_file, + (filename, data), + "creating %s" % filename, + self.verbose >= 1, + self.dry_run, + ) def list_message_files(package, suffix=".mo"): @@ -258,8 +280,10 @@ def list_message_files(package, suffix=".mo"): # basename (without extension) is a locale name localename = os.path.splitext(os.path.basename(fname))[0] domainname = "%s.mo" % package.lower() - yield (fname, os.path.join( - "share", "locale", localename, "LC_MESSAGES", domainname)) + yield ( + fname, + os.path.join("share", "locale", localename, "LC_MESSAGES", domainname), + ) class MyClean(clean): @@ -278,87 +302,89 @@ class MyClean(clean): # scripts -scripts = ['linkchecker'] +scripts = ["linkchecker"] myname = "Bastian Kleineidam" myemail = "bastian.kleineidam@web.de" data_files = [ - ('share/linkchecker', - ['config/linkcheckerrc']), - ('share/linkchecker/examples', - ['cgi-bin/lconline/leer.html.en', - 'cgi-bin/lconline/leer.html.de', - 'cgi-bin/lconline/index.html', - 'cgi-bin/lconline/lc_cgi.html.en', - 'cgi-bin/lconline/lc_cgi.html.de', - 'cgi-bin/lconline/check.js', - 'cgi-bin/lc.wsgi', - 'config/linkchecker.apache2.conf', - ]), + ("share/linkchecker", ["config/linkcheckerrc"]), + ( + "share/linkchecker/examples", + [ + "cgi-bin/lconline/leer.html.en", + "cgi-bin/lconline/leer.html.de", + "cgi-bin/lconline/index.html", + "cgi-bin/lconline/lc_cgi.html.en", + "cgi-bin/lconline/lc_cgi.html.de", + "cgi-bin/lconline/check.js", + "cgi-bin/lc.wsgi", + "config/linkchecker.apache2.conf", + ], + ), ] for (src, dst) in list_message_files(AppName): data_files.append((dst, [src])) -if os.name == 'posix': - data_files.append(('share/man/man1', ['doc/en/linkchecker.1'])) - data_files.append(('share/man/man5', ['doc/en/linkcheckerrc.5'])) - data_files.append(('share/man/de/man1', ['doc/de/linkchecker.1'])) - data_files.append(('share/man/de/man5', ['doc/de/linkcheckerrc.5'])) - data_files.append(('share/linkchecker/examples', - ['config/linkchecker-completion', - 'doc/examples/check_blacklist.sh', - 'doc/examples/check_for_x_errors.sh', - 'doc/examples/check_urls.sh'])) +if os.name == "posix": + data_files.append(("share/man/man1", ["doc/en/linkchecker.1"])) + data_files.append(("share/man/man5", ["doc/en/linkcheckerrc.5"])) + data_files.append(("share/man/de/man1", ["doc/de/linkchecker.1"])) + data_files.append(("share/man/de/man5", ["doc/de/linkcheckerrc.5"])) + data_files.append( + ( + "share/linkchecker/examples", + [ + "config/linkchecker-completion", + "doc/examples/check_blacklist.sh", + "doc/examples/check_for_x_errors.sh", + "doc/examples/check_urls.sh", + ], + ) + ) setup( - name = AppName, - version = AppVersion, - description = Description, - keywords = "link,url,site,checking,crawling,verification,validation", - author = myname, - author_email = myemail, - maintainer = myname, - maintainer_email = myemail, - url = "https://linkchecker.github.io/linkchecker/", - license = "GPL", - long_description = get_long_description(), - distclass = MyDistribution, - cmdclass = { - 'install_lib': MyInstallLib, - 'install_data': MyInstallData, - 'clean': MyClean, + name=AppName, + version=AppVersion, + description=Description, + keywords="link,url,site,checking,crawling,verification,validation", + author=myname, + author_email=myemail, + maintainer=myname, + maintainer_email=myemail, + url="https://linkchecker.github.io/linkchecker/", + license="GPL", + long_description=get_long_description(), + distclass=MyDistribution, + cmdclass={ + "install_lib": MyInstallLib, + "install_data": MyInstallData, + "clean": MyClean, }, - packages = find_packages(include=["linkcheck", "linkcheck.*"]), - scripts = scripts, - data_files = data_files, - classifiers = [ - 'Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking', - 'Development Status :: 5 - Production/Stable', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Programming Language :: Python', + packages=find_packages(include=["linkcheck", "linkcheck.*"]), + scripts=scripts, + data_files=data_files, + classifiers=[ + "Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking", + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: GNU General Public License (GPL)", + "Programming Language :: Python", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", ], - options = { - }, + options={}, # Requirements, usable with setuptools or the new Python packaging module. - python_requires = '>= 3.5', - install_requires = [ - 'requests >= 2.4', - 'dnspython', - 'beautifulsoup4', - 'pyxdg', - ], + python_requires=">= 3.5", + install_requires=["requests >= 2.4", "dnspython", "beautifulsoup4", "pyxdg",], # Commented out since they are untested and not officially supported. # See also doc/install.txt for more detailed dependency documentation. - #extra_requires = { + # extra_requires = { # "IP country info": ['GeoIP'], # http://www.maxmind.com/app/python # "GNOME proxies": ['pygtk'], # http://www.pygtk.org/downloads.html # "Bash completion": ['argcomplete'], # https://pypi.python.org/pypi/argcomplete # "Memory debugging": ['meliae'], # https://launchpad.net/meliae - #} + # } )