Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-18 03:21:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #421 from cjmayo/doc_n_scripts

Browse source 
Resolve flake8 violations in doc/ and scripts/ and enable checking of all files
This commit is contained in:
Marius Gedminas 2020-05-27 12:26:19 +03:00 committed by GitHub
commit 16f12d00d5
8 changed files with 76 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
doc/examples/filter_xml_output.py
View file

@ -28,21 +28,24 @@ def main(args):
with open(filename) as fd:
tree = parse(fd)
filter_tree(tree)
tree.write(sys.stdout, encoding='utf-8')
tree.write(sys.stdout, encoding="utf-8")
def filter_tree(tree):
"""Filter all 401 errors."""
to_remove = []
for elem in tree.findall('urldata'):
valid = elem.find('valid')
if valid is not None and valid.text == '0' and \
valid.attrib.get('result', '').startswith('401'):
for elem in tree.findall("urldata"):
valid = elem.find("valid")
if (
valid is not None
and valid.text == "0"
and valid.attrib.get("result", "").startswith("401")
):
to_remove.append(elem)
root = tree.getroot()
for elem in to_remove:
root.remove(elem)
if __name__ == '__main__':
if __name__ == "__main__":
main(sys.argv[1:])

8
doc/web/hooks/__hooks__.py
View file

@ -28,12 +28,12 @@ def chmod(config):
output_dir = config["output_dir"]
for dirpath, dirnames, filenames in os.walk(output_dir):
for dirname in dirnames:
os.chmod(os.path.join(dirpath, dirname), 0755)
os.chmod(os.path.join(dirpath, dirname), 0o755)
for filename in filenames:
os.chmod(os.path.join(dirpath, filename), 0644)
os.chmod(os.path.join(dirpath, filename), 0o644)
hooks = {
'site.output.post': [compress_javascript, compress_css],
'site.done': [chmod],
"site.output.post": [compress_javascript, compress_css],
"site.done": [chmod],
}

33
scripts/analyze_memdump.py
View file

@ -23,6 +23,7 @@ import codecs
import html
from linkcheck import strformat
def main(filename):
om = print_memorydump(filename)
dirname, basename = os.path.split(filename)
@ -32,13 +33,16 @@ def main(filename):
os.mkdir(basedir)
write_htmlfiles(om, basedir)
def print_memorydump(filename):
from meliae import loader
om = loader.load(filename, collapse=True)
om.remove_expensive_references()
print om.summarize()
print(om.summarize())
return om
def write_htmlfiles(om, basedir):
om.compute_parents()
open_files = {}
@ -47,21 +51,24 @@ def write_htmlfiles(om, basedir):
write_html_obj(fp, obj, om.objs)
close_files(open_files)
def get_file(type_str, open_files, basedir):
"""Get already opened file, or open and initialize a new one."""
if type_str not in open_files:
filename = type_str+".html"
encoding = 'utf-8'
fd = codecs.open(os.path.join(basedir, filename), 'w', encoding)
filename = type_str + ".html"
encoding = "utf-8"
fd = codecs.open(os.path.join(basedir, filename), "w", encoding)
open_files[type_str] = fd
write_html_header(fd, type_str, encoding)
return open_files[type_str]
def close_files(open_files):
for fp in open_files.values():
write_html_footer(fp)
fp.close()
HtmlHeader = """
<!doctype html>
<head>
@ -70,10 +77,15 @@ HtmlHeader = """
<body>
"""
def write_html_header(fp, type_str, encoding):
fp.write(HtmlHeader % encoding)
fp.write("<h1>Type %s</h1>\n" % type_str)
fp.write("<table><tr><th>Address</th><th>Name</th><th>Size</th><th>Parents</th><th>References</th></tr>\n")
fp.write(
"<table><tr><th>Address</th><th>Name</th><th>Size</th><th>Parents</th>"
"<th>References</th></tr>\n"
)
def get_children(obj, objs):
res = []
@ -89,6 +101,7 @@ def get_children(obj, objs):
res.append(entry)
return res
def get_parents(obj, objs):
res = []
for address in obj.parents:
@ -103,6 +116,7 @@ def get_parents(obj, objs):
res.append(entry)
return res
def write_html_obj(fp, obj, objs):
if obj.value is None:
value = "None"
@ -115,11 +129,16 @@ def write_html_obj(fp, obj, objs):
parents=",".join(get_parents(obj, objs)),
value=value,
)
fp.write("<tr><td>%(address)d</td><td>%(value)s</td><td>%(size)s</td><td>%(children)s</td></tr>\n" % attrs)
fp.write(
"<tr><td>%(address)d</td><td>%(value)s</td><td>%(size)s</td>"
"<td>%(children)s</td></tr>\n" % attrs
)
def write_html_footer(fp):
fp.write("</table></body></html>")
if __name__ == '__main__':
if __name__ == "__main__":
filename = sys.argv[1]
main(filename)

4
scripts/removeafter.py
View file

@ -6,6 +6,7 @@ from __future__ import print_function
import fileinput
import sys
def main(args):
"""Remove lines after marker."""
filename = args[0]
@ -15,5 +16,6 @@ def main(args):
if line.startswith(marker):
break
if __name__ == '__main__':
if __name__ == "__main__":
main(sys.argv[1:])

47
scripts/update_iana_uri_schemes.py
View file

@ -5,17 +5,23 @@ import requests
iana_uri_schemes = "https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml"
# CSV format: URI Scheme,Template,Description,Reference
csv_iana_uri_schemes_permanent = 'https://www.iana.org/assignments/uri-schemes/uri-schemes-1.csv'
csv_iana_uri_schemes_provisional = 'https://www.iana.org/assignments/uri-schemes/uri-schemes-2.csv'
csv_iana_uri_schemes_historical = 'https://www.iana.org/assignments/uri-schemes/uri-schemes-3.csv'
csv_iana_uri_schemes_permanent = (
"https://www.iana.org/assignments/uri-schemes/uri-schemes-1.csv"
)
csv_iana_uri_schemes_provisional = (
"https://www.iana.org/assignments/uri-schemes/uri-schemes-2.csv"
)
csv_iana_uri_schemes_historical = (
"https://www.iana.org/assignments/uri-schemes/uri-schemes-3.csv"
)
iana_uri_schemes_permanent = {}
iana_uri_schemes_provisional = {}
iana_uri_schemes_historical = {}
iana_uri_schemes_other = {
"clsid": "Microsoft specific",
"find" : "Mozilla specific",
"isbn" : "ISBN (int. book numbers)",
"clsid": "Microsoft specific",
"find": "Mozilla specific",
"isbn": "ISBN (int. book numbers)",
"javascript": "JavaScript",
}
@ -58,33 +64,38 @@ ignored_schemes_re = re.compile(ignored_schemes, re.VERBOSE)
is_unknown_scheme = ignored_schemes_re.match
'''
def main(args):
parse_csv_file(csv_iana_uri_schemes_permanent, iana_uri_schemes_permanent)
parse_csv_file(csv_iana_uri_schemes_provisional, iana_uri_schemes_provisional)
parse_csv_file(csv_iana_uri_schemes_historical, iana_uri_schemes_historical)
for scheme in iana_uri_schemes_other:
if (scheme in iana_uri_schemes_permanent or
scheme in iana_uri_schemes_provisional or
scheme in iana_uri_schemes_historical):
if (
scheme in iana_uri_schemes_permanent
or scheme in iana_uri_schemes_provisional
or scheme in iana_uri_schemes_historical
):
raise ValueError(scheme)
for scheme in filter_uri_schemes_permanent:
if scheme in iana_uri_schemes_permanent:
del iana_uri_schemes_permanent[scheme]
args = dict(
uri = iana_uri_schemes,
permanent = get_regex(iana_uri_schemes_permanent),
provisional = get_regex(iana_uri_schemes_provisional),
historical = get_regex(iana_uri_schemes_historical),
other = get_regex(iana_uri_schemes_other),
uri=iana_uri_schemes,
permanent=get_regex(iana_uri_schemes_permanent),
provisional=get_regex(iana_uri_schemes_provisional),
historical=get_regex(iana_uri_schemes_historical),
other=get_regex(iana_uri_schemes_other),
)
res = template % args
print res
print(res)
return 0
def get_regex(schemes):
expr = ["|%s # %s" % (re.escape(scheme).ljust(10), description)
for scheme, description in sorted(schemes.items())]
expr = [
"|%s # %s" % (re.escape(scheme).ljust(10), description)
for scheme, description in sorted(schemes.items())
]
return "\n".join(expr)
@ -102,5 +113,5 @@ def parse_csv_file(url, res):
res[scheme] = description
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))

6
scripts/viewprof.py
View file

@ -7,6 +7,7 @@ Usage: $0 <filename>
import sys
import yappi
def main(args):
filename = args[0]
stats = yappi.YFuncStats()
@ -14,6 +15,5 @@ def main(args):
stats.print_all()
if __name__ == '__main__':
main(sys.argv[1:])
if __name__ == "__main__":
main(sys.argv[1:])

2
setup.cfg
View file

@ -25,7 +25,7 @@ filename =
builtins =
_
_n
max-line-length = 80
max-line-length = 88
per-file-ignores =
# In several files imports intentionally cause:
# E402: module level import not at top of file

2
tox.ini
View file

@ -35,7 +35,7 @@ deps =
[testenv:flake8]
deps = flake8
skip_install = true
commands = flake8 linkchecker setup.py linkcheck {posargs}
commands = flake8 {posargs}
[testenv:check-manifest]
deps = check-manifest