Initial commit

.gitignore

@@ -0,0 +1,6 @@
+node_modules
+build
+*.log
+package-lock.json
+target
+Cargo.lock

Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "tree-sitter-htmldjango"
+description = "htmldjango grammar for the tree-sitter parsing library"
+version = "0.0.1"
+keywords = ["incremental", "parsing", "htmldjango"]
+categories = ["parsing", "text-editors"]
+repository = "https://github.com/interdependence/tree-sitter-htmldjango"
+edition = "2018"
+license = "MIT"
+
+build = "bindings/rust/build.rs"
+include = [
+  "bindings/rust/*",
+  "grammar.js",
+  "queries/*",
+  "src/*",
+]
+
+[lib]
+path = "bindings/rust/lib.rs"
+
+[dependencies]
+tree-sitter = "~0.20"
+
+[build-dependencies]
+cc = "1.0"

README.md

@@ -0,0 +1,8 @@
+# tree-sitter-htmldjango
+
+Django template language grammar for [tree-sitter][].
+
+[tree-sitter]: https://github.com/tree-sitter/tree-sitter
+
+#### References
+* https://docs.djangoproject.com/en/4.0/ref/templates/language/

binding.gyp

@@ -0,0 +1,18 @@
+{
+  "targets": [
+    {
+      "target_name": "tree_sitter_htmldjango_binding",
+      "include_dirs": [
+        "<!(node -e \"require('nan')\")",
+        "src"
+      ],
+      "sources": [
+        "bindings/node/binding.cc",
+        "src/parser.c",
+      ],
+      "cflags_c": [
+        "-std=c99",
+      ]
+    }
+  ]
+}

bindings/node/binding.cc

@@ -0,0 +1,28 @@
+#include "tree_sitter/parser.h"
+#include <node.h>
+#include "nan.h"
+
+using namespace v8;
+
+extern "C" TSLanguage * tree_sitter_htmldjango();
+
+namespace {
+
+NAN_METHOD(New) {}
+
+void Init(Local<Object> exports, Local<Object> module) {
+  Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
+  tpl->SetClassName(Nan::New("Language").ToLocalChecked());
+  tpl->InstanceTemplate()->SetInternalFieldCount(1);
+
+  Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
+  Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
+  Nan::SetInternalFieldPointer(instance, 0, tree_sitter_htmldjango());
+
+  Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("htmldjango").ToLocalChecked());
+  Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
+}
+
+NODE_MODULE(tree_sitter_htmldjango_binding, Init)
+
+}  // namespace

bindings/node/index.js

@@ -0,0 +1,19 @@
+try {
+  module.exports = require("../../build/Release/tree_sitter_htmldjango_binding");
+} catch (error1) {
+  if (error1.code !== 'MODULE_NOT_FOUND') {
+    throw error1;
+  }
+  try {
+    module.exports = require("../../build/Debug/tree_sitter_htmldjango_binding");
+  } catch (error2) {
+    if (error2.code !== 'MODULE_NOT_FOUND') {
+      throw error2;
+    }
+    throw error1
+  }
+}
+
+try {
+  module.exports.nodeTypeInfo = require("../../src/node-types.json");
+} catch (_) {}

bindings/rust/build.rs

@@ -0,0 +1,40 @@
+fn main() {
+    let src_dir = std::path::Path::new("src");
+
+    let mut c_config = cc::Build::new();
+    c_config.include(&src_dir);
+    c_config
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-Wno-unused-but-set-variable")
+        .flag_if_supported("-Wno-trigraphs");
+    let parser_path = src_dir.join("parser.c");
+    c_config.file(&parser_path);
+
+    // If your language uses an external scanner written in C,
+    // then include this block of code:
+
+    /*
+    let scanner_path = src_dir.join("scanner.c");
+    c_config.file(&scanner_path);
+    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
+    */
+
+    c_config.compile("parser");
+    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
+
+    // If your language uses an external scanner written in C++,
+    // then include this block of code:
+
+    /*
+    let mut cpp_config = cc::Build::new();
+    cpp_config.cpp(true);
+    cpp_config.include(&src_dir);
+    cpp_config
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-Wno-unused-but-set-variable");
+    let scanner_path = src_dir.join("scanner.cc");
+    cpp_config.file(&scanner_path);
+    cpp_config.compile("scanner");
+    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
+    */
+}

bindings/rust/lib.rs

@@ -0,0 +1,52 @@
+//! This crate provides htmldjango language support for the [tree-sitter][] parsing library.
+//!
+//! Typically, you will use the [language][language func] function to add this language to a
+//! tree-sitter [Parser][], and then use the parser to parse some code:
+//!
+//! ```
+//! let code = "";
+//! let mut parser = tree_sitter::Parser::new();
+//! parser.set_language(tree_sitter_htmldjango::language()).expect("Error loading htmldjango grammar");
+//! let tree = parser.parse(code, None).unwrap();
+//! ```
+//!
+//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+//! [language func]: fn.language.html
+//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
+//! [tree-sitter]: https://tree-sitter.github.io/
+
+use tree_sitter::Language;
+
+extern "C" {
+    fn tree_sitter_htmldjango() -> Language;
+}
+
+/// Get the tree-sitter [Language][] for this grammar.
+///
+/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+pub fn language() -> Language {
+    unsafe { tree_sitter_htmldjango() }
+}
+
+/// The content of the [`node-types.json`][] file for this grammar.
+///
+/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
+pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
+
+// Uncomment these to include any queries that this grammar contains
+
+// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
+// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
+// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
+// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn test_can_load_grammar() {
+        let mut parser = tree_sitter::Parser::new();
+        parser
+            .set_language(super::language())
+            .expect("Error loading htmldjango language");
+    }
+}

grammar.js

@@ -0,0 +1,152 @@
+module.exports = grammar({
+  name: "htmldjango",
+
+  // Handle whitespace explicitly
+  extras: $ => [],
+
+  rules: {
+    template: $ => repeat(
+      $._node
+    ),
+
+    _node: $ => choice(
+      $._expression,
+      $._statement,
+      $._comment,
+      $.content,
+      /\s+/
+    ),
+
+    // General rules
+    keyword: $ => choice("on", "off", "with", "as", "silent", "only", "from", "random", "by"),
+    operator: $ => choice("==", "!=", "<", ">", "<=", ">="),
+    keyword_operator: $ => choice("and", "or", "not", "in", "not in", "is", "is not"),
+    number: $ => repeat1(/[0-9]/),
+    boolean: $ => choice("True", "False"),
+    string: $ => seq(
+      choice(
+        seq("'", repeat(/[^']/), "'"),
+        seq('"', repeat(/[^"]/), '"')
+      ),
+      repeat(seq("|", $.filter))
+    ),
+
+    _word: $ => repeat1(/[A-Za-z0-9_]/),
+    _ws: $ => repeat1(" "),
+
+    // Expressions
+    _expression: $ => seq("{{", optional($._ws), $.variable, optional($._ws), "}}"),
+
+    variable: $ => seq($.variable_name, repeat(seq("|", $.filter))),
+    // Django variables cannot start with an "_", can contain one or two words separated by a "."
+    variable_name: $ => seq(repeat1(/[A-Za-z]/), optional($._word), optional(seq(".", $._word))),
+   
+    filter: $ => seq($.filter_name, optional(seq(":", choice($.filter_argument, $._quoted_filter_argument)))),
+    filter_name: $ => $._word,
+    filter_argument: $ => $._word,
+    _quoted_filter_argument: $ => choice(
+      seq("'", alias(repeat(/[^']/), $.filter_argument), "'"),
+      seq('"', alias(repeat(/[^"]/), $.filter_argument), '"')
+    ),
+   
+    // Statements
+    // unpaired type {% tag %}
+    // paired type   {% tag %}..{% endtag %}
+    _statement: $ => choice(
+      $.paired_statement,
+      alias($.if_statement, $.paired_statement),
+      alias($.for_statement, $.paired_statement),
+      alias($.filter_statement, $.paired_statement),
+      $.unpaired_statement,
+      $.detatched_end_statement
+    ),
+
+    paired_statement: $ => {
+      const tag_names = [
+        "autoescape",
+        "block",
+        "blocktranslate",
+        "ifchanged",
+        "spaceless",
+        "verbatim",
+        "with"
+      ];
+
+      return choice(...tag_names.map((tag_name) => seq(
+        "{%", $._ws, alias(tag_name, $.tag_name), $._ws, repeat($._attribute), "%}",
+        repeat($._node),
+        "{%", $._ws, "end", alias(tag_name, $.tag_name), $._ws, repeat($._attribute), alias("%}", $.end_paired_statement))));
+    },
+
+    if_statement: $ => seq(
+      "{%", $._ws, alias("if", $.tag_name), $._ws, repeat($._attribute), "%}",
+      repeat($._node),
+      repeat(prec.left(seq(
+        alias($.elif_statement, $.branch_statement),
+        repeat($._node),
+      ))),
+      optional(seq(
+        alias($.else_statement, $.branch_statement),
+        repeat($._node),
+      )),
+      "{%", $._ws, "end", alias("if", $.tag_name), $._ws, alias("%}", $.end_paired_statement)
+    ),
+    elif_statement: $ => seq("{%", $._ws, alias("elif", $.tag_name), $._ws, repeat($._attribute), "%}"),
+    else_statement: $ => seq("{%", $._ws, alias("else", $.tag_name), $._ws, "%}"),
+
+    for_statement: $ => seq(
+      "{%", $._ws, alias("for", $.tag_name), $._ws, repeat($._attribute), "%}",
+      repeat($._node),
+      optional(seq(
+        alias($.empty_statement, $.branch_statement),
+        repeat($._node),
+      )),
+      "{%", $._ws, "end", alias("for", $.tag_name), $._ws, alias("%}", $.end_paired_statement)
+    ),
+    empty_statement: $ => seq("{%", $._ws, alias("empty", $.tag_name), $._ws, repeat($._attribute), "%}"),
+
+    filter_statement: $ => seq(
+      "{%", $._ws, alias("filter", $.tag_name), $._ws, $.filter, repeat(seq("|", $.filter)), $._ws, "%}",
+      repeat($._node),
+      "{%", $._ws, "end", alias("filter", $.tag_name), $._ws, alias("%}", $.end_paired_statement)
+    ),
+    
+    unpaired_statement: $ => seq("{%", $._ws, alias($._word, $.tag_name), $._ws, repeat($._attribute), "%}"),
+    detatched_end_statement: $ => seq("{%", $._ws, "end", alias($._word, $.tag_name), $._ws, repeat($._attribute), "%}"),
+
+    _attribute: $ => seq(
+      choice(
+        $.keyword,
+        $.operator,
+        $.keyword_operator,
+        $.number,
+        $.boolean,
+        $.string,
+        $.variable
+      ),
+      choice(
+        $._ws,
+        seq(optional($._ws), ",", optional($._ws)),
+        seq(optional($._ws), "=", optional($._ws))
+      )
+    ),
+
+    // Comments
+    // unpaired type {# comment #}
+    // paired type   {% comment optional_label %}..{% endcomment %}
+    _comment: $ => choice(
+      $.unpaired_comment,
+      $.paired_comment
+    ),
+    unpaired_comment: $ => seq("{#", repeat(/.|\s/), repeat(seq(alias($.unpaired_comment, ""), repeat(/.|\s/))), "#}"),
+    paired_comment: $ => seq(
+      alias("{%", ""), $._ws, "comment", optional(seq($._ws, $._word)), $._ws, alias("%}", ""),
+      repeat(/.|\s/),
+      repeat(seq(alias($.paired_comment, ""), repeat(/.|\s/))),
+      alias("{%", ""), $._ws, "endcomment", $._ws, alias("%}", "")
+    ),
+
+    // All other content
+    content: $ => /([^\{]|\{[^{%#])+/
+  }
+});

package.json

@@ -0,0 +1,28 @@
+{
+  "name": "tree-sitter-htmldjango",
+  "version": "0.1.0",
+  "description": "A tree-sitter grammar for the Django template language",
+  "main": "bindings/node",
+  "scripts": {
+    "test": "tree-sitter test"
+  },
+  "keywords": [
+    "django",
+    "html",
+    "tree-sitter",
+    "parser",
+    "grammar"
+  ],
+  "author": "William Vandervalk",
+  "license": "MIT",
+  "dependencies": {
+    "nan": "^2.15.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/interdependence/tree-sitter-htmldjango.git"
+  },
+  "devDependencies": {
+    "tree-sitter-cli": "^0.20.4"
+  }
+}

queries/folds.scm

@@ -0,0 +1,4 @@
+[
+  (paired_statement)
+  (paired_comment)
+] @fold

queries/highlights.scm

@@ -0,0 +1,25 @@
+[
+  (unpaired_comment)
+  (paired_comment)
+] @comment
+
+[
+  "{{"
+  "}}"
+  "{%"
+  "%}"
+  (end_paired_statement)
+] @tag
+
+"end" @keyword.return
+
+(variable_name) @variable
+(filter_name) @method
+(filter_argument) @parameter
+(tag_name) @function
+(keyword) @keyword
+(operator) @operator
+(keyword_operator) @keyword.operator
+(number) @number
+(boolean) @boolean
+(string) @string

queries/indents.scm

@@ -0,0 +1,3 @@
+(paired_statement) @indent
+(end_paired_statement) @indent_end
+(branch_statement) @branch

queries/injections.scm

@@ -0,0 +1,3 @@
+((content) @injection.content
+ (#set! injection.language "html")
+ (#set! injection.combined))

src/node-types.json

@@ -0,0 +1,538 @@
+[
+  {
+    "type": "",
+    "named": false,
+    "fields": {}
+  },
+  {
+    "type": "boolean",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "branch_statement",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "boolean",
+          "named": true
+        },
+        {
+          "type": "keyword",
+          "named": true
+        },
+        {
+          "type": "keyword_operator",
+          "named": true
+        },
+        {
+          "type": "number",
+          "named": true
+        },
+        {
+          "type": "operator",
+          "named": true
+        },
+        {
+          "type": "string",
+          "named": true
+        },
+        {
+          "type": "tag_name",
+          "named": true
+        },
+        {
+          "type": "variable",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "detatched_end_statement",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "boolean",
+          "named": true
+        },
+        {
+          "type": "keyword",
+          "named": true
+        },
+        {
+          "type": "keyword_operator",
+          "named": true
+        },
+        {
+          "type": "number",
+          "named": true
+        },
+        {
+          "type": "operator",
+          "named": true
+        },
+        {
+          "type": "string",
+          "named": true
+        },
+        {
+          "type": "tag_name",
+          "named": true
+        },
+        {
+          "type": "variable",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "filter",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "filter_argument",
+          "named": true
+        },
+        {
+          "type": "filter_name",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "filter_argument",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "filter_name",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "keyword",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "keyword_operator",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "number",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "operator",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "paired_comment",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "paired_statement",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "boolean",
+          "named": true
+        },
+        {
+          "type": "branch_statement",
+          "named": true
+        },
+        {
+          "type": "content",
+          "named": true
+        },
+        {
+          "type": "detatched_end_statement",
+          "named": true
+        },
+        {
+          "type": "end_paired_statement",
+          "named": true
+        },
+        {
+          "type": "filter",
+          "named": true
+        },
+        {
+          "type": "keyword",
+          "named": true
+        },
+        {
+          "type": "keyword_operator",
+          "named": true
+        },
+        {
+          "type": "number",
+          "named": true
+        },
+        {
+          "type": "operator",
+          "named": true
+        },
+        {
+          "type": "paired_comment",
+          "named": true
+        },
+        {
+          "type": "paired_statement",
+          "named": true
+        },
+        {
+          "type": "string",
+          "named": true
+        },
+        {
+          "type": "tag_name",
+          "named": true
+        },
+        {
+          "type": "unpaired_comment",
+          "named": true
+        },
+        {
+          "type": "unpaired_statement",
+          "named": true
+        },
+        {
+          "type": "variable",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "string",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "filter",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "tag_name",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "template",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "content",
+          "named": true
+        },
+        {
+          "type": "detatched_end_statement",
+          "named": true
+        },
+        {
+          "type": "paired_comment",
+          "named": true
+        },
+        {
+          "type": "paired_statement",
+          "named": true
+        },
+        {
+          "type": "unpaired_comment",
+          "named": true
+        },
+        {
+          "type": "unpaired_statement",
+          "named": true
+        },
+        {
+          "type": "variable",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "unpaired_comment",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "unpaired_statement",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "boolean",
+          "named": true
+        },
+        {
+          "type": "keyword",
+          "named": true
+        },
+        {
+          "type": "keyword_operator",
+          "named": true
+        },
+        {
+          "type": "number",
+          "named": true
+        },
+        {
+          "type": "operator",
+          "named": true
+        },
+        {
+          "type": "string",
+          "named": true
+        },
+        {
+          "type": "tag_name",
+          "named": true
+        },
+        {
+          "type": "variable",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "variable",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "filter",
+          "named": true
+        },
+        {
+          "type": "variable_name",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "variable_name",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "",
+    "named": false
+  },
+  {
+    "type": " ",
+    "named": false
+  },
+  {
+    "type": "!=",
+    "named": false
+  },
+  {
+    "type": "\"",
+    "named": false
+  },
+  {
+    "type": "#}",
+    "named": false
+  },
+  {
+    "type": "%}",
+    "named": false
+  },
+  {
+    "type": "'",
+    "named": false
+  },
+  {
+    "type": ",",
+    "named": false
+  },
+  {
+    "type": ".",
+    "named": false
+  },
+  {
+    "type": ":",
+    "named": false
+  },
+  {
+    "type": "<",
+    "named": false
+  },
+  {
+    "type": "<=",
+    "named": false
+  },
+  {
+    "type": "=",
+    "named": false
+  },
+  {
+    "type": "==",
+    "named": false
+  },
+  {
+    "type": ">",
+    "named": false
+  },
+  {
+    "type": ">=",
+    "named": false
+  },
+  {
+    "type": "False",
+    "named": false
+  },
+  {
+    "type": "True",
+    "named": false
+  },
+  {
+    "type": "and",
+    "named": false
+  },
+  {
+    "type": "as",
+    "named": false
+  },
+  {
+    "type": "by",
+    "named": false
+  },
+  {
+    "type": "comment",
+    "named": false
+  },
+  {
+    "type": "content",
+    "named": true
+  },
+  {
+    "type": "end",
+    "named": false
+  },
+  {
+    "type": "end_paired_statement",
+    "named": true
+  },
+  {
+    "type": "endcomment",
+    "named": false
+  },
+  {
+    "type": "from",
+    "named": false
+  },
+  {
+    "type": "in",
+    "named": false
+  },
+  {
+    "type": "is",
+    "named": false
+  },
+  {
+    "type": "is not",
+    "named": false
+  },
+  {
+    "type": "not",
+    "named": false
+  },
+  {
+    "type": "not in",
+    "named": false
+  },
+  {
+    "type": "off",
+    "named": false
+  },
+  {
+    "type": "on",
+    "named": false
+  },
+  {
+    "type": "only",
+    "named": false
+  },
+  {
+    "type": "or",
+    "named": false
+  },
+  {
+    "type": "random",
+    "named": false
+  },
+  {
+    "type": "silent",
+    "named": false
+  },
+  {
+    "type": "with",
+    "named": false
+  },
+  {
+    "type": "{#",
+    "named": false
+  },
+  {
+    "type": "{%",
+    "named": false
+  },
+  {
+    "type": "{{",
+    "named": false
+  },
+  {
+    "type": "|",
+    "named": false
+  },
+  {
+    "type": "}}",
+    "named": false
+  }
+]

src/tree_sitter/parser.h

@@ -0,0 +1,224 @@
+#ifndef TREE_SITTER_PARSER_H_
+#define TREE_SITTER_PARSER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define ts_builtin_sym_error ((TSSymbol)-1)
+#define ts_builtin_sym_end 0
+#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
+
+typedef uint16_t TSStateId;
+
+#ifndef TREE_SITTER_API_H_
+typedef uint16_t TSSymbol;
+typedef uint16_t TSFieldId;
+typedef struct TSLanguage TSLanguage;
+#endif
+
+typedef struct {
+  TSFieldId field_id;
+  uint8_t child_index;
+  bool inherited;
+} TSFieldMapEntry;
+
+typedef struct {
+  uint16_t index;
+  uint16_t length;
+} TSFieldMapSlice;
+
+typedef struct {
+  bool visible;
+  bool named;
+  bool supertype;
+} TSSymbolMetadata;
+
+typedef struct TSLexer TSLexer;
+
+struct TSLexer {
+  int32_t lookahead;
+  TSSymbol result_symbol;
+  void (*advance)(TSLexer *, bool);
+  void (*mark_end)(TSLexer *);
+  uint32_t (*get_column)(TSLexer *);
+  bool (*is_at_included_range_start)(const TSLexer *);
+  bool (*eof)(const TSLexer *);
+};
+
+typedef enum {
+  TSParseActionTypeShift,
+  TSParseActionTypeReduce,
+  TSParseActionTypeAccept,
+  TSParseActionTypeRecover,
+} TSParseActionType;
+
+typedef union {
+  struct {
+    uint8_t type;
+    TSStateId state;
+    bool extra;
+    bool repetition;
+  } shift;
+  struct {
+    uint8_t type;
+    uint8_t child_count;
+    TSSymbol symbol;
+    int16_t dynamic_precedence;
+    uint16_t production_id;
+  } reduce;
+  uint8_t type;
+} TSParseAction;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+} TSLexMode;
+
+typedef union {
+  TSParseAction action;
+  struct {
+    uint8_t count;
+    bool reusable;
+  } entry;
+} TSParseActionEntry;
+
+struct TSLanguage {
+  uint32_t version;
+  uint32_t symbol_count;
+  uint32_t alias_count;
+  uint32_t token_count;
+  uint32_t external_token_count;
+  uint32_t state_count;
+  uint32_t large_state_count;
+  uint32_t production_id_count;
+  uint32_t field_count;
+  uint16_t max_alias_sequence_length;
+  const uint16_t *parse_table;
+  const uint16_t *small_parse_table;
+  const uint32_t *small_parse_table_map;
+  const TSParseActionEntry *parse_actions;
+  const char * const *symbol_names;
+  const char * const *field_names;
+  const TSFieldMapSlice *field_map_slices;
+  const TSFieldMapEntry *field_map_entries;
+  const TSSymbolMetadata *symbol_metadata;
+  const TSSymbol *public_symbol_map;
+  const uint16_t *alias_map;
+  const TSSymbol *alias_sequences;
+  const TSLexMode *lex_modes;
+  bool (*lex_fn)(TSLexer *, TSStateId);
+  bool (*keyword_lex_fn)(TSLexer *, TSStateId);
+  TSSymbol keyword_capture_token;
+  struct {
+    const bool *states;
+    const TSSymbol *symbol_map;
+    void *(*create)(void);
+    void (*destroy)(void *);
+    bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
+    unsigned (*serialize)(void *, char *);
+    void (*deserialize)(void *, const char *, unsigned);
+  } external_scanner;
+  const TSStateId *primary_state_ids;
+};
+
+/*
+ *  Lexer Macros
+ */
+
+#define START_LEXER()           \
+  bool result = false;          \
+  bool skip = false;            \
+  bool eof = false;             \
+  int32_t lookahead;            \
+  goto start;                   \
+  next_state:                   \
+  lexer->advance(lexer, skip);  \
+  start:                        \
+  skip = false;                 \
+  lookahead = lexer->lookahead;
+
+#define ADVANCE(state_value) \
+  {                          \
+    state = state_value;     \
+    goto next_state;         \
+  }
+
+#define SKIP(state_value) \
+  {                       \
+    skip = true;          \
+    state = state_value;  \
+    goto next_state;      \
+  }
+
+#define ACCEPT_TOKEN(symbol_value)     \
+  result = true;                       \
+  lexer->result_symbol = symbol_value; \
+  lexer->mark_end(lexer);
+
+#define END_STATE() return result;
+
+/*
+ *  Parse Table Macros
+ */
+
+#define SMALL_STATE(id) id - LARGE_STATE_COUNT
+
+#define STATE(id) id
+
+#define ACTIONS(id) id
+
+#define SHIFT(state_value)            \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = state_value            \
+    }                                 \
+  }}
+
+#define SHIFT_REPEAT(state_value)     \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = state_value,           \
+      .repetition = true              \
+    }                                 \
+  }}
+
+#define SHIFT_EXTRA()                 \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .extra = true                   \
+    }                                 \
+  }}
+
+#define REDUCE(symbol_val, child_count_val, ...) \
+  {{                                             \
+    .reduce = {                                  \
+      .type = TSParseActionTypeReduce,           \
+      .symbol = symbol_val,                      \
+      .child_count = child_count_val,            \
+      __VA_ARGS__                                \
+    },                                           \
+  }}
+
+#define RECOVER()                    \
+  {{                                 \
+    .type = TSParseActionTypeRecover \
+  }}
+
+#define ACCEPT_INPUT()              \
+  {{                                \
+    .type = TSParseActionTypeAccept \
+  }}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PARSER_H_

test/corpus/comments.txt

@@ -0,0 +1,33 @@
+==================
+comment
+==================
+
+<html>
+  {# comment #}
+  content
+  {% comment optional_label %}
+    content
+  {% endcomment %}
+  content
+  {# {{ variable }} {% statement %} {# comment #} content #}
+  content
+  {% comment optional_label %}
+  {% comment optional_label %}
+  {% comment optional_label %}
+  {% endcomment %}
+  {% endcomment %}
+  {% endcomment %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_comment)
+  (content)
+  (paired_comment)
+  (content)
+  (unpaired_comment)
+  (content)
+  (paired_comment)
+  (content))

test/corpus/expressions.txt

@@ -0,0 +1,43 @@
+==================
+variable
+==================
+
+<html>
+  {{ variable }}
+  {{ variable|filter }}
+  {{ variable|filter:argument }}
+  {{ variable|filter:"quoted argument" }}
+  {{ variable|filter|filter:argument|filter:"quoted argument" }}
+</html>
+
+---
+
+(template
+  (content)
+  (variable
+    (variable_name))
+  (variable
+    (variable_name)
+    (filter
+      (filter_name)))
+  (variable
+    (variable_name)
+    (filter
+      (filter_name)
+      (filter_argument)))
+  (variable
+    (variable_name)
+    (filter
+      (filter_name)
+      (filter_argument)))
+  (variable
+    (variable_name)
+    (filter
+      (filter_name))
+    (filter
+      (filter_name)
+      (filter_argument))
+    (filter
+      (filter_name)
+      (filter_argument)))
+  (content))

test/corpus/statements.txt

@@ -0,0 +1,611 @@
+==================
+csrf_token
+==================
+
+<html>
+  {% csrf_token %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name))
+  (content))
+
+==================
+cycle
+==================
+
+<html>
+  {% cycle 'row1' rowvalue2 'row3' as row silent %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (string)
+    (variable
+      (variable_name))
+    (string)
+    (keyword)
+    (variable
+      (variable_name))
+    (keyword))
+  (content))
+
+==================
+debug
+==================
+
+<html>
+  {% debug %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name))
+  (content))
+
+==================
+extends
+==================
+
+<html>
+  {% extends "template.html" %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (string))
+  (content))
+
+==================
+firstof
+==================
+
+<html>
+  {% firstof var1 var2 var3 "fallback value" %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (variable
+      (variable_name))
+    (variable
+      (variable_name))
+    (variable
+      (variable_name))
+    (string))
+  (content))
+
+==================
+include
+==================
+
+<html>
+  {% include "template.html" with var="value" only %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (string)
+    (keyword)
+    (variable
+      (variable_name))
+    (string)
+    (keyword))
+  (content))
+
+==================
+load
+==================
+
+<html>
+  {% load foo bar from somelibrary.otherlibrary %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (variable
+      (variable_name))
+    (variable
+      (variable_name))
+    (keyword)
+    (variable
+      (variable_name)))
+  (content))
+
+==================
+lorem
+==================
+
+<html>
+  {% lorem 2 w random %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (number)
+    (variable
+      (variable_name))
+    (keyword))
+  (content))
+
+==================
+now
+==================
+
+<html>
+  {% now "jS F Y H:i" %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (string))
+  (content))
+
+==================
+regroup
+==================
+
+<html>
+  {% regroup list by attribute as list_attribute %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (variable
+      (variable_name))
+    (keyword)
+    (variable
+      (variable_name))
+    (keyword)
+    (variable
+      (variable_name)))
+  (content))
+
+==================
+resetcycle
+==================
+
+<html>
+  {% resetcycle name %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (variable
+      (variable_name)))
+  (content))
+
+==================
+templatetag
+==================
+
+<html>
+  {% templatetag openblock %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (variable
+      (variable_name)))
+  (content))
+
+==================
+url
+==================
+
+<html>
+  {% url 'some-url-name' arg arg2 as the_url %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (string)
+    (variable
+      (variable_name))
+    (variable
+      (variable_name))
+    (keyword)
+    (variable
+      (variable_name)))
+  (content))
+
+==================
+widthratio
+==================
+
+<html>
+  {% widthratio this_value max_value max_width %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (variable
+      (variable_name))
+    (variable
+      (variable_name))
+    (variable
+      (variable_name)))
+  (content))
+
+==================
+get_static_prefix
+==================
+
+<html>
+  {% get_static_prefix %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name))
+  (content))
+
+==================
+get_media_prefix
+==================
+
+<html>
+  {% get_media_prefix %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name))
+  (content))
+
+==================
+if
+==================
+
+<html>
+  {% if variable %}
+    content
+  {% elif variable2 %}
+    content
+  {% elif variable3 %}
+    content
+  {% else %}
+    content
+  {% endif %}
+</html>
+
+---
+
+(template
+  (content)
+  (paired_statement
+    (tag_name)
+    (variable
+      (variable_name))
+    (content)
+    (branch_statement
+      (tag_name)
+      (variable
+        (variable_name)))
+    (content)
+    (branch_statement
+      (tag_name)
+      (variable
+        (variable_name)))
+    (content)
+    (branch_statement
+      (tag_name))
+    (content)
+    (tag_name)
+    (end_paired_statement))
+  (content))
+
+==================
+for
+==================
+
+<html>
+  {% for variable in list %}
+    {{ variable }}
+  {% empty %}
+    content
+  {% endfor %}
+</html>
+
+---
+
+(template
+  (content)
+  (paired_statement
+    (tag_name)
+    (variable
+      (variable_name))
+    (keyword_operator)
+    (variable
+      (variable_name))
+    (variable
+      (variable_name))
+    (branch_statement
+      (tag_name))
+    (content)
+    (tag_name)
+    (end_paired_statement))
+  (content))
+
+==================
+block
+==================
+
+<html>
+  {% block name %}
+    content
+  {% endblock %}
+</html>
+
+---
+
+(template
+  (content)
+  (paired_statement
+    (tag_name)
+    (variable
+      (variable_name))
+    (content)
+    (tag_name)
+    (end_paired_statement))
+  (content))
+
+==================
+autoescape
+==================
+
+<html>
+  {% autoescape off %}
+    content
+  {% endautoescape %}
+</html>
+
+---
+
+(template
+  (content)
+  (paired_statement
+    (tag_name)
+    (keyword)
+    (content)
+    (tag_name)
+    (end_paired_statement))
+  (content))
+
+==================
+filter
+==================
+
+<html>
+  {% filter filter1|filter2 %}
+    content
+  {% endfilter %}
+</html>
+
+---
+
+(template
+  (content)
+  (paired_statement
+    (tag_name)
+    (filter
+      (filter_name))
+    (filter
+      (filter_name))
+    (content)
+    (tag_name)
+    (end_paired_statement))
+  (content))
+
+==================
+ifchanged
+==================
+
+<html>
+  {% for variable in list %}
+    {% ifchanged variable %}
+      content
+    {% endifchanged %}
+  {% endfor %}
+</html>
+
+---
+
+(template
+  (content)
+  (paired_statement
+    (tag_name)
+    (variable
+      (variable_name))
+    (keyword_operator)
+    (variable
+      (variable_name))
+    (paired_statement
+      (tag_name)
+      (variable
+        (variable_name))
+      (content)
+      (tag_name)
+      (end_paired_statement))
+    (tag_name)
+    (end_paired_statement))
+  (content))
+
+==================
+blocktranslate
+==================
+
+<html>
+  {% blocktranslate %}
+    content
+  {% endblocktranslate %}
+</html>
+
+---
+
+(template
+  (content)
+  (paired_statement
+    (tag_name)
+    (content)
+    (tag_name)
+    (end_paired_statement))
+  (content))
+
+==================
+spaceless
+==================
+
+<html>
+  {% spaceless %}
+    content
+  {% endspaceless %}
+</html>
+
+---
+
+(template
+  (content)
+  (paired_statement
+    (tag_name)
+    (content)
+    (tag_name)
+    (end_paired_statement))
+  (content))
+
+==================
+verbatim
+==================
+
+<html>
+  {% verbatim block %}
+    content
+  {% endverbatim block %}
+</html>
+
+---
+
+(template
+  (content)
+  (paired_statement
+    (tag_name)
+    (variable
+      (variable_name))
+    (content)
+    (tag_name)
+    (variable
+      (variable_name))
+    (end_paired_statement))
+  (content))
+
+==================
+with
+==================
+
+<html>
+  {% with variable1=1 variable2=2 %}
+    content
+  {% endwith %}
+</html>
+
+---
+
+(template
+  (content)
+  (paired_statement
+    (tag_name)
+    (variable
+      (variable_name))
+    (number)
+    (variable
+      (variable_name))
+    (number)
+    (content)
+    (tag_name)
+    (end_paired_statement))
+  (content))
+
+==================
+custom
+==================
+
+<html>
+  {% unpaired %}
+  {% paired %}
+    content
+  {% endpaired %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name))
+  (unpaired_statement
+    (tag_name))
+  (content)
+  (detatched_end_statement
+      (tag_name))
+  (content))