added reformatter

This commit is contained in:
Christopher Pickering 2021-07-23 15:58:24 -05:00
parent c554a795e9
commit cc0510780c
No known key found for this signature in database
GPG key ID: E14DB3B0A0FACF84
9 changed files with 494 additions and 62 deletions

View file

@ -1,6 +1,6 @@
# djlint
Simple Django template linter.
Simple Django template linter and reformatter.
[![codecov](https://codecov.io/gh/Riverside-Healthcare/djlint/branch/master/graph/badge.svg?token=eNTG721BAA)](https://codecov.io/gh/Riverside-Healthcare/djlint)
[![test](https://github.com/Riverside-Healthcare/djlint/actions/workflows/test.yml/badge.svg)](https://github.com/Riverside-Healthcare/djlint/actions/workflows/test.yml)
@ -12,18 +12,31 @@ Simple Django template linter.
```sh
pip install djlint
```
## Usage
## Linter Usage
```sh
djlint <file or path>
djlint src # file or path
```
## Reformatter Usage
Reforamtting is beta. Check the output before applying changes. Please PR any changes needed 👍🏽
```sh
djlint src --reformat --check
djlint src --reformat
```
## Optional args
| Arg | Definition | Default |
|:----|:-----------|:--------|
-e, --extension | File extension to lint. | default=html
--check | Checks file formatting |
--reformat | Reformats html |
## Rules
## Linter Rules
### Error Codes

View file

@ -5,10 +5,6 @@ build-backend = "setuptools.build_meta"
[tool]
[tool.flake8]
max-line-length = 99
extend-ignore = "D103, F401"
[tool.black]
max_line_length = 99
@ -23,3 +19,7 @@ use_parentheses = true
ensure_newline_before_comments = true
line_length = 99
quiet = true
[tool.pylint.messages_control]
disable = "E1120, R0914, E0401, R0912"

6
setup.cfg Normal file
View file

@ -0,0 +1,6 @@
[mypy]
ignore_missing_imports = True
[flake8]
max-line-length = 99
extend-ignore = "D103, F401"

View file

@ -6,60 +6,25 @@ usage::
djlint INPUT -e <extension>
options:
--check | will check html formatting for needed changes
--reformat | will reformat html
"""
import os
import re
import sys
from concurrent.futures import ProcessPoolExecutor
from functools import partial
from pathlib import Path
import click
import yaml
from click import echo
from colorama import Fore, Style, deinit, init
rules = yaml.load(
(Path(__file__).parent / "rules.yaml").read_text(encoding="utf8"),
Loader=yaml.SafeLoader,
)
def get_line(start, line_ends):
"""Get the line number and index of match."""
# print(start, line_ends)
line = list(filter(lambda pair: pair["end"] > start, line_ends))[0]
return "%d:%d" % (line_ends.index(line) + 1, start - line["start"])
def lint_file(this_file: Path):
"""Check file for formatting errors."""
file_name = str(this_file)
errors: dict = {file_name: []}
html = this_file.read_text(encoding="utf8")
# build list of line ends for file
line_ends = [
{"start": m.start(), "end": m.end()}
for m in re.finditer(r"(?:.*\n)|(?:[^\n]+$)", html)
]
for rule in rules:
rule = rule["rule"]
for pattern in rule["patterns"]:
for match in re.finditer(pattern, html, re.DOTALL):
errors[file_name].append(
{
"code": rule["name"],
"line": get_line(match.start(), line_ends),
"match": match.group()[:20].strip(),
"message": rule["message"],
}
)
return errors
from djlint.lint import lint_file
from djlint.reformat import reformat_file
def get_src(src: Path, extension=None):
@ -73,7 +38,7 @@ def get_src(src: Path, extension=None):
paths = list(src.glob(r"**/*.%s" % extension))
if len(paths) == 0:
echo(Fore.BLUE + "No files to lint! 😢")
echo(Fore.BLUE + "No files to check! 😢")
return []
return paths
@ -108,6 +73,55 @@ def build_output(error):
return len(errors)
def build_check_output(errors, quiet):
"""Build output for reformat check."""
if len(errors) == 0:
return 0
color = {"-": Fore.YELLOW, "+": Fore.GREEN, "@": Style.BRIGHT + Fore.BLUE}
if quiet is True or len(list(errors.values())[0]) == 0:
echo(
Fore.GREEN
+ Style.BRIGHT
+ str(list(errors.keys())[0])
+ Style.DIM
+ Style.RESET_ALL
)
else:
echo(
"{}\n{}\n{}===============================".format(
Fore.GREEN + Style.BRIGHT, list(errors.keys())[0], Style.DIM
)
+ Style.RESET_ALL
)
for diff in list(errors.values())[0]:
echo(
"{}{}{}".format(
color.get(diff[:1], Style.RESET_ALL), diff, Style.RESET_ALL
),
err=False,
)
return len(list(filter(lambda x: len(x) > 0, errors.values())))
def build_quantity(size: int):
"""Count files in a list."""
return "%d file%s" % (size, ("s" if size > 1 else ""))
def build_quantity_tense(size: int):
"""Count files in a list."""
return "%d file%s %s" % (
size,
("s" if size > 1 or size == 0 else ""),
("were" if size > 1 or size == 0 else "was"),
)
@click.command(context_settings={"help_option_names": ["-h", "--help"]})
@click.argument(
"src",
@ -125,16 +139,39 @@ def build_output(error):
help="File extension to lint",
show_default=True,
)
def main(src: str, extension: str):
@click.option(
"--reformat",
is_flag=True,
help="Reformat the file.",
)
@click.option(
"--check",
is_flag=True,
help="Reformat the file.",
)
@click.option(
"--quiet",
is_flag=True,
help="Reformat the file.",
)
def main(src: str, extension: str, reformat: bool, check: bool, quiet: bool):
"""Djlint django template files."""
file_list = get_src(Path(src), extension)
if len(file_list) == 0:
return
file_quantity = "%d file%s" % (len(file_list), ("s" if len(file_list) > 1 else ""))
file_quantity = build_quantity(len(file_list))
echo("\nChecking %s!" % file_quantity)
message = "Reformatt" if reformat is True else "Lint"
echo(
"%sing %s!\n"
% (
message,
file_quantity,
)
)
worker_count = os.cpu_count()
@ -143,17 +180,34 @@ def main(src: str, extension: str):
worker_count = min(worker_count, 60)
with ProcessPoolExecutor(max_workers=worker_count) as exe:
file_errors = exe.map(lint_file, file_list)
if reformat is True:
func = partial(reformat_file, check)
file_errors = exe.map(func, file_list)
else:
file_errors = exe.map(lint_file, file_list)
# format errors
success_message = ""
error_count = 0
for error in file_errors:
error_count += build_output(error)
success_message = "Checked %s, found %d errors." % (
file_quantity,
error_count,
)
if reformat is not True:
for error in file_errors:
error_count += build_output(error)
success_message = "%sed %s, found %d errors." % (
message,
file_quantity,
error_count,
)
else:
for error in file_errors:
error_count += build_check_output(error, quiet)
tense_message = (
build_quantity(error_count) + " would be"
if check is True
else build_quantity_tense(error_count)
)
success_message = "%s updated." % tense_message
echo("\n%s\n" % (success_message))

View file

@ -1,3 +1,4 @@
"""Djlint access through python -m djlint."""
from djlint import main
main()

46
src/djlint/lint.py Normal file
View file

@ -0,0 +1,46 @@
"""Djlint html linter."""
import re
from pathlib import Path
import yaml
rules = yaml.load(
(Path(__file__).parent / "rules.yaml").read_text(encoding="utf8"),
Loader=yaml.SafeLoader,
)
def get_line(start, line_ends):
"""Get the line number and index of match."""
line = list(filter(lambda pair: pair["end"] > start, line_ends))[0]
return "%d:%d" % (line_ends.index(line) + 1, start - line["start"])
def lint_file(this_file: Path):
"""Check file for formatting errors."""
file_name = str(this_file)
errors: dict = {file_name: []}
html = this_file.read_text(encoding="utf8")
# build list of line ends for file
line_ends = [
{"start": m.start(), "end": m.end()}
for m in re.finditer(r"(?:.*\n)|(?:[^\n]+$)", html)
]
for rule in rules:
rule = rule["rule"]
for pattern in rule["patterns"]:
for match in re.finditer(pattern, html, re.DOTALL):
errors[file_name].append(
{
"code": rule["name"],
"line": get_line(match.start(), line_ends),
"match": match.group()[:20].strip(),
"message": rule["message"],
}
)
return errors

233
src/djlint/reformat.py Normal file
View file

@ -0,0 +1,233 @@
"""Djlint reformat html files.
Much code is borrowed from https://github.com/rareyman/HTMLBeautify, many thanks!
"""
import difflib
import re
from pathlib import Path
from djlint.settings import settings
indent = settings.get("indent")
tag_indent = settings.get("tag_indent")
tag_unindent = settings.get("tag_unindent")
tag_unindent_line = settings.get("tag_unindent_line")
tag_pos_inline = settings.get("tag_pos_inline")
reduce_extralines_gt = settings.get("reduce_extralines_gt")
max_line_length = settings.get("max_line_length")
format_long_attributes = settings.get("format_long_attributes")
ignored_tag_opening = settings.get("ignored_tag_opening")
ignored_tag_closing = settings.get("ignored_tag_closing")
tag_newline_before = settings.get("tag_newline_before")
tag_newline_after = settings.get("tag_newline_after")
tag_raw_flat_opening = settings.get("tag_raw_flat_opening")
tag_raw_flat_closing = settings.get("tag_raw_flat_closing")
attribute_pattern = settings.get("attribute_pattern")
tag_pattern = settings.get("tag_pattern")
def clean_line(line):
"""Clean up a line of html.
* remove duplicate spaces
* remove trailing spaces
"""
return re.sub(r" {2,}", " ", line.strip())
def flatten_attributes(match):
"""Flatten multiline attributes back to one line."""
return "{} {}{}".format(
match.group(1),
" ".join(match.group(2).strip().splitlines()),
match.group(3),
)
def format_attributes(match):
"""Spread long attributes over multiple lines."""
leading_space = match.group(1)
tag = match.group(2)
attributes = "{}{}".format(
("\n" + leading_space + indent),
("\n" + leading_space + indent).join(
re.findall(attribute_pattern, match.group(3).strip())
),
)
close = match.group(4)
return "{}{}{}{}".format(
leading_space,
tag,
attributes,
close,
)
def remove_indentation(rawcode):
"""Remove indentation from raw code."""
rawcode_flat = ""
is_block_ignored = False
is_block_raw = False
for item in rawcode.strip().splitlines():
# ignore raw code
if re.search(tag_raw_flat_closing, item, re.IGNORECASE):
tmp = clean_line(item)
is_block_raw = False
elif re.search(tag_raw_flat_opening, item, re.IGNORECASE):
tmp = clean_line(item)
is_block_raw = True
# find ignored blocks and retain indentation, otherwise strip white space
if re.search(ignored_tag_closing, item, re.IGNORECASE):
tmp = clean_line(item)
is_block_ignored = False
elif re.search(ignored_tag_opening, item, re.IGNORECASE):
tmp = item
is_block_ignored = True
# not filtered so just output it
elif is_block_raw:
# remove tabs from raw_flat content
tmp = re.sub(indent, "", item)
elif is_block_ignored:
tmp = item
else:
tmp = item.strip()
rawcode_flat = rawcode_flat + tmp + "\n"
# put attributes back on one line
rawcode_flat = re.sub(
tag_pattern,
flatten_attributes,
rawcode_flat,
flags=re.IGNORECASE | re.DOTALL | re.MULTILINE,
)
# add missing line breaks before tag
rawcode_flat = re.sub(
tag_newline_before,
r"\1\n\2",
rawcode_flat,
flags=re.IGNORECASE | re.DOTALL | re.MULTILINE,
)
# add missing line breaks after tag
rawcode_flat = re.sub(
tag_newline_after, r"\1\n\2", rawcode_flat, flags=re.IGNORECASE | re.MULTILINE
)
return rawcode_flat
def add_indentation(rawcode):
"""Indent raw code."""
rawcode_flat_list = re.split("\n", rawcode)
beautified_code = ""
indent_level = 0
is_block_raw = False
blank_counter = 0
for item in rawcode_flat_list:
# if a one-line, inline tag, just process it
if re.search(tag_pos_inline, item, re.IGNORECASE):
tmp = (indent * indent_level) + item
blank_counter = 0
# if unindent, move left
elif re.search(tag_unindent, item, re.IGNORECASE):
indent_level = indent_level - 1
tmp = (indent * indent_level) + item
blank_counter = 0
elif re.search(tag_unindent_line, item, re.IGNORECASE):
tmp = (indent * (indent_level - 1)) + item
blank_counter = 0
# if indent, move right
elif re.search(tag_indent, item, re.IGNORECASE):
tmp = (indent * indent_level) + item
indent_level = indent_level + 1
blank_counter = 0
# if raw, flatten! no indenting!
elif tag_raw_flat_opening and re.search(
tag_raw_flat_opening, item, re.IGNORECASE
):
tmp = item
is_block_raw = True
blank_counter = 0
elif tag_raw_flat_closing and re.search(
tag_raw_flat_closing, item, re.IGNORECASE
):
tmp = item
is_block_raw = False
blank_counter = 0
elif is_block_raw is True:
tmp = item
# if just a blank line
elif item.strip() == "":
if blank_counter < int(reduce_extralines_gt) or blank_counter + 1:
tmp = item.strip()
# otherwise, just leave same level
else:
tmp = item # (indent * indent_level) + item
beautified_code = beautified_code + tmp + "\n"
if format_long_attributes:
# find lines longer than x
new_beautified = ""
for line in beautified_code.splitlines():
if len(line) > max_line_length:
# get leading space, and attributes
line = re.sub(r"(\s*?)(<\w+)(.+?)(/?>)", format_attributes, line)
new_beautified += "\n" + line
beautified_code = new_beautified
return beautified_code.strip() + "\n"
def reformat_file(check: bool, this_file: Path):
"""Reformat html file."""
rawcode = this_file.read_text(encoding="utf8")
beautified_code = add_indentation(remove_indentation(rawcode))
if check is not True:
# update the file
this_file.write_text(beautified_code)
out = {
this_file: list(
difflib.unified_diff(rawcode.splitlines(), beautified_code.splitlines())
)
}
return out

37
src/djlint/settings.py Normal file
View file

@ -0,0 +1,37 @@
"""Settings for reformater."""
# pylint: disable=C0301
# flake8: noqa
settings: dict = {
# default indentation
"indent": " ",
# indicates tags whose contents should not be formatted
"ignored_tag_opening": r"<script|<style|<!--|{\*|<\?php|<pre",
# indicates when to stop ignoring
"ignored_tag_closing": r"</script|</style|-->|\*}|\?>|</pre",
# the contents of these tag blocks will be start on a new line
"tag_newline_before": r"([^\n])((?:\{% +?(?:if|end|for|block|endblock|else|spaceless|compress|load|include)|(?:<script)|(?:</?(html|head|body|div|a|nav|ul|ol|dl|li|table|thead|tbody|tr|th|td|blockquote|select|form|option|optgroup|fieldset|legend|label|header|main|section|aside|footer|figure|video|span|p|g|svg|button|h\d))))",
# these tags should be followed by a newline
"tag_newline_after": r"((?:\{% +?(?:if|end|for|block|else|spaceless|compress|load|include)(?:.*?%}))|(?:<html|<head|</head|<body|</body|</script|<div|</div|<nav|</nav|<ul|</ul|<ol|</ol|<dl|</dl|<li|</li|<table|</table|<thead|</thead|<tbody|</tbody|<tr|</tr|<th|</th|<td|</td|<blockquote|</blockquote|<select|</select|<form|</form|<option|</option|<optgroup|</optgroup|<fieldset|</fieldset|<legend|</legend|<label|</label|<header|</header|<main|</main|<section|</section|<aside|</aside|<footer|</footer|<figure|</figure|<video|</video|</span|<p|</p|<g|</g|<svg|</svg|</h\d).*?\>)([^\n])",
# the contents of these tag blocks will be indented
"tag_indent": r"\{% +?(if|for|block|else|spaceless|compress)|(?:<(?:html|head|body|div|a|nav|ul|ol|dl|li|table|thead|tbody|tr|th|td|blockquote|select|form|option|optgroup|fieldset|legend|label|header|main|section|aside|footer|figure|video|span|p|g|svg|h\d|button))",
# this signals when tags should be unindented (see tags above)
"tag_unindent": r"\{% end|(?:</(?:html|head|body|div|a|nav|ul|ol|dl|li|table|thead|tbody|tr|th|td|blockquote|select|form|option|optgroup|fieldset|legend|label|header|main|section|aside|footer|figure|video|span|p|g|svg|h\d|button))",
# these tags should be unindented and next line will be indented
"tag_unindent_line": r"\{% el",
# these tags can sometimes be on one line
"tag_pos_inline": r"\{% if.*endif %\}|\{% block.*endblock %\}|<link.*/>|<link.*\">|<link.*>|<meta.*/>|<script.*</script>|<div.*</div>|<a.*</a>|<li.*</li>|<dt.*</dt>|<dd.*</dd>|<th.*</th>|<td.*</td>|<legend.*</legend>|<label.*</label>|<option.*</option>|<input.*/>|<input.*\">|<span.*</span>|<p.*</p>|<path.*/>|<!--.*-->|<button.*</button>",
# these tags use raw code and should flatten to column 1
# tabs will be removed inside these tags! use spaces for spacing if needed!
# flatten starting with this tag...
"tag_raw_flat_opening": r"",
# ...stop flattening when you encounter this tag
"tag_raw_flat_closing": r"",
# reduce empty lines greater than x to 1 line
"reduce_extralines_gt": 2,
# if lines are longer than x
"max_line_length": 99,
"format_long_attributes": True,
# pattern used to find attributes in a tag
"attribute_pattern": r"([a-zA-Z-_]+=(?:\".*?\"|\'.*?\')|required|checked)\s*",
"tag_pattern": r"(<\w+?)((?:\n\s*?[^>]+?)+?)(/?\>)",
}

42
tox.ini
View file

@ -6,6 +6,48 @@ envlist =
skip_missing_interpreters = True
isolated_build = True
[testenv:isort]
deps = isort
commands = isort src/djlint
skip_install: true
[testenv:black]
deps = black
commands = black src/djlint
skip_install: true
[testenv:lint]
deps =
reformat
flake8
flake8-bugbear
flake8-docstrings
flake8-rst-docstrings
flake8-rst
flake8-builtins
pep8-naming
flake8-comprehensions
flake8-bandit
flake8-eradicate
flake8-pytest-style
flake8-print
flake8-simplify
flake8-variables-names
flake8-markdown
pygments
black
pylint
mypy
types-PyYAML
commands =
flake8 src/djlint
black --fast --check src/djlint
pylint src/djlint
mypy src/djlint
skip_install: true
[testenv:clean]
skip_install: true
deps = coverage