diff --git a/README.md b/README.md index d622590..cca33da 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # djlint -Simple Django template linter. +Simple Django template linter and reformatter. [![codecov](https://codecov.io/gh/Riverside-Healthcare/djlint/branch/master/graph/badge.svg?token=eNTG721BAA)](https://codecov.io/gh/Riverside-Healthcare/djlint) [![test](https://github.com/Riverside-Healthcare/djlint/actions/workflows/test.yml/badge.svg)](https://github.com/Riverside-Healthcare/djlint/actions/workflows/test.yml) @@ -12,18 +12,31 @@ Simple Django template linter. ```sh pip install djlint ``` -## Usage +## Linter Usage ```sh -djlint +djlint src # file or path ``` + +## Reformatter Usage + +Reforamtting is beta. Check the output before applying changes. Please PR any changes needed 👍🏽 + +```sh +djlint src --reformat --check + +djlint src --reformat +``` + ## Optional args | Arg | Definition | Default | |:----|:-----------|:--------| -e, --extension | File extension to lint. | default=html +--check | Checks file formatting | +--reformat | Reformats html | -## Rules +## Linter Rules ### Error Codes diff --git a/pyproject.toml b/pyproject.toml index 481cdbc..1d3b2ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,10 +5,6 @@ build-backend = "setuptools.build_meta" [tool] -[tool.flake8] -max-line-length = 99 -extend-ignore = "D103, F401" - [tool.black] max_line_length = 99 @@ -23,3 +19,7 @@ use_parentheses = true ensure_newline_before_comments = true line_length = 99 quiet = true + + +[tool.pylint.messages_control] +disable = "E1120, R0914, E0401, R0912" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..6edfd3e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[mypy] +ignore_missing_imports = True + +[flake8] +max-line-length = 99 +extend-ignore = "D103, F401" diff --git a/src/djlint/__init__.py b/src/djlint/__init__.py index ac85ab9..f77ebd7 100644 --- a/src/djlint/__init__.py +++ b/src/djlint/__init__.py @@ -6,60 +6,25 @@ usage:: djlint INPUT -e + options: + + --check | will check html formatting for needed changes + --reformat | will reformat html + """ import os -import re import sys from concurrent.futures import ProcessPoolExecutor +from functools import partial from pathlib import Path import click -import yaml from click import echo from colorama import Fore, Style, deinit, init -rules = yaml.load( - (Path(__file__).parent / "rules.yaml").read_text(encoding="utf8"), - Loader=yaml.SafeLoader, -) - - -def get_line(start, line_ends): - """Get the line number and index of match.""" - # print(start, line_ends) - line = list(filter(lambda pair: pair["end"] > start, line_ends))[0] - - return "%d:%d" % (line_ends.index(line) + 1, start - line["start"]) - - -def lint_file(this_file: Path): - """Check file for formatting errors.""" - file_name = str(this_file) - errors: dict = {file_name: []} - html = this_file.read_text(encoding="utf8") - - # build list of line ends for file - line_ends = [ - {"start": m.start(), "end": m.end()} - for m in re.finditer(r"(?:.*\n)|(?:[^\n]+$)", html) - ] - - for rule in rules: - rule = rule["rule"] - - for pattern in rule["patterns"]: - for match in re.finditer(pattern, html, re.DOTALL): - errors[file_name].append( - { - "code": rule["name"], - "line": get_line(match.start(), line_ends), - "match": match.group()[:20].strip(), - "message": rule["message"], - } - ) - - return errors +from djlint.lint import lint_file +from djlint.reformat import reformat_file def get_src(src: Path, extension=None): @@ -73,7 +38,7 @@ def get_src(src: Path, extension=None): paths = list(src.glob(r"**/*.%s" % extension)) if len(paths) == 0: - echo(Fore.BLUE + "No files to lint! 😢") + echo(Fore.BLUE + "No files to check! 😢") return [] return paths @@ -108,6 +73,55 @@ def build_output(error): return len(errors) +def build_check_output(errors, quiet): + """Build output for reformat check.""" + if len(errors) == 0: + return 0 + + color = {"-": Fore.YELLOW, "+": Fore.GREEN, "@": Style.BRIGHT + Fore.BLUE} + + if quiet is True or len(list(errors.values())[0]) == 0: + echo( + Fore.GREEN + + Style.BRIGHT + + str(list(errors.keys())[0]) + + Style.DIM + + Style.RESET_ALL + ) + + else: + echo( + "{}\n{}\n{}===============================".format( + Fore.GREEN + Style.BRIGHT, list(errors.keys())[0], Style.DIM + ) + + Style.RESET_ALL + ) + + for diff in list(errors.values())[0]: + echo( + "{}{}{}".format( + color.get(diff[:1], Style.RESET_ALL), diff, Style.RESET_ALL + ), + err=False, + ) + + return len(list(filter(lambda x: len(x) > 0, errors.values()))) + + +def build_quantity(size: int): + """Count files in a list.""" + return "%d file%s" % (size, ("s" if size > 1 else "")) + + +def build_quantity_tense(size: int): + """Count files in a list.""" + return "%d file%s %s" % ( + size, + ("s" if size > 1 or size == 0 else ""), + ("were" if size > 1 or size == 0 else "was"), + ) + + @click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument( "src", @@ -125,16 +139,39 @@ def build_output(error): help="File extension to lint", show_default=True, ) -def main(src: str, extension: str): +@click.option( + "--reformat", + is_flag=True, + help="Reformat the file.", +) +@click.option( + "--check", + is_flag=True, + help="Reformat the file.", +) +@click.option( + "--quiet", + is_flag=True, + help="Reformat the file.", +) +def main(src: str, extension: str, reformat: bool, check: bool, quiet: bool): """Djlint django template files.""" file_list = get_src(Path(src), extension) if len(file_list) == 0: return - file_quantity = "%d file%s" % (len(file_list), ("s" if len(file_list) > 1 else "")) + file_quantity = build_quantity(len(file_list)) - echo("\nChecking %s!" % file_quantity) + message = "Reformatt" if reformat is True else "Lint" + + echo( + "%sing %s!\n" + % ( + message, + file_quantity, + ) + ) worker_count = os.cpu_count() @@ -143,17 +180,34 @@ def main(src: str, extension: str): worker_count = min(worker_count, 60) with ProcessPoolExecutor(max_workers=worker_count) as exe: - file_errors = exe.map(lint_file, file_list) + if reformat is True: + func = partial(reformat_file, check) + file_errors = exe.map(func, file_list) + else: + file_errors = exe.map(lint_file, file_list) # format errors + success_message = "" error_count = 0 - for error in file_errors: - error_count += build_output(error) - success_message = "Checked %s, found %d errors." % ( - file_quantity, - error_count, - ) + if reformat is not True: + for error in file_errors: + error_count += build_output(error) + + success_message = "%sed %s, found %d errors." % ( + message, + file_quantity, + error_count, + ) + else: + for error in file_errors: + error_count += build_check_output(error, quiet) + tense_message = ( + build_quantity(error_count) + " would be" + if check is True + else build_quantity_tense(error_count) + ) + success_message = "%s updated." % tense_message echo("\n%s\n" % (success_message)) diff --git a/src/djlint/__main__.py b/src/djlint/__main__.py index 04567da..acc39a3 100644 --- a/src/djlint/__main__.py +++ b/src/djlint/__main__.py @@ -1,3 +1,4 @@ +"""Djlint access through python -m djlint.""" from djlint import main main() diff --git a/src/djlint/lint.py b/src/djlint/lint.py new file mode 100644 index 0000000..ea57e55 --- /dev/null +++ b/src/djlint/lint.py @@ -0,0 +1,46 @@ +"""Djlint html linter.""" +import re +from pathlib import Path + +import yaml + +rules = yaml.load( + (Path(__file__).parent / "rules.yaml").read_text(encoding="utf8"), + Loader=yaml.SafeLoader, +) + + +def get_line(start, line_ends): + """Get the line number and index of match.""" + line = list(filter(lambda pair: pair["end"] > start, line_ends))[0] + + return "%d:%d" % (line_ends.index(line) + 1, start - line["start"]) + + +def lint_file(this_file: Path): + """Check file for formatting errors.""" + file_name = str(this_file) + errors: dict = {file_name: []} + html = this_file.read_text(encoding="utf8") + + # build list of line ends for file + line_ends = [ + {"start": m.start(), "end": m.end()} + for m in re.finditer(r"(?:.*\n)|(?:[^\n]+$)", html) + ] + + for rule in rules: + rule = rule["rule"] + + for pattern in rule["patterns"]: + for match in re.finditer(pattern, html, re.DOTALL): + errors[file_name].append( + { + "code": rule["name"], + "line": get_line(match.start(), line_ends), + "match": match.group()[:20].strip(), + "message": rule["message"], + } + ) + + return errors diff --git a/src/djlint/reformat.py b/src/djlint/reformat.py new file mode 100644 index 0000000..c9db09a --- /dev/null +++ b/src/djlint/reformat.py @@ -0,0 +1,233 @@ +"""Djlint reformat html files. + +Much code is borrowed from https://github.com/rareyman/HTMLBeautify, many thanks! +""" + +import difflib +import re +from pathlib import Path + +from djlint.settings import settings + +indent = settings.get("indent") +tag_indent = settings.get("tag_indent") +tag_unindent = settings.get("tag_unindent") + +tag_unindent_line = settings.get("tag_unindent_line") + +tag_pos_inline = settings.get("tag_pos_inline") + +reduce_extralines_gt = settings.get("reduce_extralines_gt") + +max_line_length = settings.get("max_line_length") +format_long_attributes = settings.get("format_long_attributes") + +ignored_tag_opening = settings.get("ignored_tag_opening") +ignored_tag_closing = settings.get("ignored_tag_closing") + +tag_newline_before = settings.get("tag_newline_before") +tag_newline_after = settings.get("tag_newline_after") + +tag_raw_flat_opening = settings.get("tag_raw_flat_opening") +tag_raw_flat_closing = settings.get("tag_raw_flat_closing") + +attribute_pattern = settings.get("attribute_pattern") +tag_pattern = settings.get("tag_pattern") + + +def clean_line(line): + """Clean up a line of html. + + * remove duplicate spaces + * remove trailing spaces + + """ + return re.sub(r" {2,}", " ", line.strip()) + + +def flatten_attributes(match): + """Flatten multiline attributes back to one line.""" + return "{} {}{}".format( + match.group(1), + " ".join(match.group(2).strip().splitlines()), + match.group(3), + ) + + +def format_attributes(match): + """Spread long attributes over multiple lines.""" + leading_space = match.group(1) + + tag = match.group(2) + + attributes = "{}{}".format( + ("\n" + leading_space + indent), + ("\n" + leading_space + indent).join( + re.findall(attribute_pattern, match.group(3).strip()) + ), + ) + + close = match.group(4) + + return "{}{}{}{}".format( + leading_space, + tag, + attributes, + close, + ) + + +def remove_indentation(rawcode): + """Remove indentation from raw code.""" + rawcode_flat = "" + is_block_ignored = False + is_block_raw = False + + for item in rawcode.strip().splitlines(): + + # ignore raw code + if re.search(tag_raw_flat_closing, item, re.IGNORECASE): + tmp = clean_line(item) + is_block_raw = False + + elif re.search(tag_raw_flat_opening, item, re.IGNORECASE): + tmp = clean_line(item) + is_block_raw = True + + # find ignored blocks and retain indentation, otherwise strip white space + if re.search(ignored_tag_closing, item, re.IGNORECASE): + tmp = clean_line(item) + is_block_ignored = False + + elif re.search(ignored_tag_opening, item, re.IGNORECASE): + tmp = item + is_block_ignored = True + + # not filtered so just output it + elif is_block_raw: + # remove tabs from raw_flat content + tmp = re.sub(indent, "", item) + + elif is_block_ignored: + tmp = item + + else: + tmp = item.strip() + + rawcode_flat = rawcode_flat + tmp + "\n" + + # put attributes back on one line + rawcode_flat = re.sub( + tag_pattern, + flatten_attributes, + rawcode_flat, + flags=re.IGNORECASE | re.DOTALL | re.MULTILINE, + ) + + # add missing line breaks before tag + rawcode_flat = re.sub( + tag_newline_before, + r"\1\n\2", + rawcode_flat, + flags=re.IGNORECASE | re.DOTALL | re.MULTILINE, + ) + + # add missing line breaks after tag + rawcode_flat = re.sub( + tag_newline_after, r"\1\n\2", rawcode_flat, flags=re.IGNORECASE | re.MULTILINE + ) + + return rawcode_flat + + +def add_indentation(rawcode): + """Indent raw code.""" + rawcode_flat_list = re.split("\n", rawcode) + + beautified_code = "" + indent_level = 0 + is_block_raw = False + blank_counter = 0 + + for item in rawcode_flat_list: + + # if a one-line, inline tag, just process it + if re.search(tag_pos_inline, item, re.IGNORECASE): + tmp = (indent * indent_level) + item + blank_counter = 0 + + # if unindent, move left + elif re.search(tag_unindent, item, re.IGNORECASE): + indent_level = indent_level - 1 + tmp = (indent * indent_level) + item + blank_counter = 0 + + elif re.search(tag_unindent_line, item, re.IGNORECASE): + tmp = (indent * (indent_level - 1)) + item + blank_counter = 0 + + # if indent, move right + elif re.search(tag_indent, item, re.IGNORECASE): + tmp = (indent * indent_level) + item + indent_level = indent_level + 1 + blank_counter = 0 + + # if raw, flatten! no indenting! + elif tag_raw_flat_opening and re.search( + tag_raw_flat_opening, item, re.IGNORECASE + ): + tmp = item + is_block_raw = True + blank_counter = 0 + + elif tag_raw_flat_closing and re.search( + tag_raw_flat_closing, item, re.IGNORECASE + ): + tmp = item + is_block_raw = False + blank_counter = 0 + + elif is_block_raw is True: + tmp = item + + # if just a blank line + elif item.strip() == "": + if blank_counter < int(reduce_extralines_gt) or blank_counter + 1: + tmp = item.strip() + + # otherwise, just leave same level + else: + tmp = item # (indent * indent_level) + item + + beautified_code = beautified_code + tmp + "\n" + + if format_long_attributes: + # find lines longer than x + new_beautified = "" + for line in beautified_code.splitlines(): + if len(line) > max_line_length: + # get leading space, and attributes + line = re.sub(r"(\s*?)(<\w+)(.+?)(/?>)", format_attributes, line) + + new_beautified += "\n" + line + beautified_code = new_beautified + + return beautified_code.strip() + "\n" + + +def reformat_file(check: bool, this_file: Path): + """Reformat html file.""" + rawcode = this_file.read_text(encoding="utf8") + + beautified_code = add_indentation(remove_indentation(rawcode)) + + if check is not True: + # update the file + this_file.write_text(beautified_code) + + out = { + this_file: list( + difflib.unified_diff(rawcode.splitlines(), beautified_code.splitlines()) + ) + } + return out diff --git a/src/djlint/settings.py b/src/djlint/settings.py new file mode 100644 index 0000000..8240c38 --- /dev/null +++ b/src/djlint/settings.py @@ -0,0 +1,37 @@ +"""Settings for reformater.""" +# pylint: disable=C0301 +# flake8: noqa +settings: dict = { + # default indentation + "indent": " ", + # indicates tags whose contents should not be formatted + "ignored_tag_opening": r"|\*}|\?>|)([^\n])", + # the contents of these tag blocks will be indented + "tag_indent": r"\{% +?(if|for|block|else|spaceless|compress)|(?:<(?:html|head|body|div|a|nav|ul|ol|dl|li|table|thead|tbody|tr|th|td|blockquote|select|form|option|optgroup|fieldset|legend|label|header|main|section|aside|footer|figure|video|span|p|g|svg|h\d|button))", + # this signals when tags should be unindented (see tags above) + "tag_unindent": r"\{% end|(?:|||||||||||||||||||||", + # these tags use raw code and should flatten to column 1 + # tabs will be removed inside these tags! use spaces for spacing if needed! + # flatten starting with this tag... + "tag_raw_flat_opening": r"", + # ...stop flattening when you encounter this tag + "tag_raw_flat_closing": r"", + # reduce empty lines greater than x to 1 line + "reduce_extralines_gt": 2, + # if lines are longer than x + "max_line_length": 99, + "format_long_attributes": True, + # pattern used to find attributes in a tag + "attribute_pattern": r"([a-zA-Z-_]+=(?:\".*?\"|\'.*?\')|required|checked)\s*", + "tag_pattern": r"(<\w+?)((?:\n\s*?[^>]+?)+?)(/?\>)", +} diff --git a/tox.ini b/tox.ini index 4510744..21f956f 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,48 @@ envlist = skip_missing_interpreters = True isolated_build = True +[testenv:isort] +deps = isort +commands = isort src/djlint +skip_install: true + + +[testenv:black] +deps = black +commands = black src/djlint +skip_install: true + +[testenv:lint] +deps = + reformat + flake8 + flake8-bugbear + flake8-docstrings + flake8-rst-docstrings + flake8-rst + flake8-builtins + pep8-naming + flake8-comprehensions + flake8-bandit + flake8-eradicate + flake8-pytest-style + flake8-print + flake8-simplify + flake8-variables-names + flake8-markdown + pygments + black + pylint + mypy + types-PyYAML +commands = + flake8 src/djlint + black --fast --check src/djlint + pylint src/djlint + mypy src/djlint +skip_install: true + + [testenv:clean] skip_install: true deps = coverage