Fix conflicts in attribute/keyword parsing

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter-htmldjango"
 description = "htmldjango grammar for the tree-sitter parsing library"
-version = "0.1.1"
+version = "1.0.0"
 keywords = ["incremental", "parsing", "htmldjango"]
 categories = ["parsing", "text-editors"]
 repository = "https://github.com/interdependence/tree-sitter-htmldjango"

grammar.js

@@ -1,8 +1,7 @@
 module.exports = grammar({
   name: "htmldjango",
 
-  // Handle whitespace explicitly
+  word: $ => $._identifier,
-  extras: $ => [],
 
   rules: {
     template: $ => repeat(
@@ -13,16 +12,39 @@ module.exports = grammar({
       $._expression,
       $._statement,
       $._comment,
-      $.content,
+      $.content
-      /\s+/
     ),
 
     // General rules
-    keyword: $ => choice("on", "off", "with", "as", "silent", "only", "from", "random", "by"),
+    keyword: $ => token(seq(
+      choice(
+        "on",
+        "off",
+        "with",
+        "as",
+        "silent",
+        "only",
+        "from",
+        "random",
+        "by"
+      ),
+      /\s/
+    )),
+    keyword_operator: $ => token(seq(
+      choice(
+        "and",
+        "or",
+        "not",
+        "in",
+        "not in",
+        "is",
+        "is not"
+      ),
+      /\s/
+    )),
     operator: $ => choice("==", "!=", "<", ">", "<=", ">="),
-    keyword_operator: $ => choice("and", "or", "not", "in", "not in", "is", "is not"),
+    number: $ => /[0-9]+/,
-    number: $ => repeat1(/[0-9]/),
+    boolean: $ => token(seq(choice("True", "False"), /\s/)),
-    boolean: $ => choice("True", "False"),
     string: $ => seq(
       choice(
         seq("'", repeat(/[^']/), "'"),
@@ -31,24 +53,23 @@ module.exports = grammar({
       repeat(seq("|", $.filter))
     ),
 
-    _word: $ => repeat1(/[A-Za-z0-9_]/),
+    _identifier: $ => /\w+/,
-    _ws: $ => repeat1(" "),
 
     // Expressions
-    _expression: $ => seq("{{", optional($._ws), $.variable, optional($._ws), "}}"),
+    _expression: $ => seq("{{", $.variable, "}}"),
 
     variable: $ => seq($.variable_name, repeat(seq("|", $.filter))),
     // Django variables cannot start with an "_", can contain one or more words separated by a "."
-    variable_name: $ => seq(repeat1(/[A-Za-z]/), optional($._word), repeat(seq(".", $._word))),
+    variable_name: $ => /[a-zA-Z](\w+)?((\.?\w)+)?/,
-   
+
     filter: $ => seq($.filter_name, optional(seq(":", choice($.filter_argument, $._quoted_filter_argument)))),
-    filter_name: $ => $._word,
+    filter_name: $ => $._identifier,
-    filter_argument: $ => seq($._word, repeat(seq(".", $._word))),
+    filter_argument: $ => seq($._identifier, repeat(seq(".", $._identifier))),
     _quoted_filter_argument: $ => choice(
       seq("'", alias(repeat(/[^']/), $.filter_argument), "'"),
       seq('"', alias(repeat(/[^"]/), $.filter_argument), '"')
     ),
-   
+
     // Statements
     // unpaired type {% tag %}
     // paired type   {% tag %}..{% endtag %}
@@ -57,8 +78,7 @@ module.exports = grammar({
       alias($.if_statement, $.paired_statement),
       alias($.for_statement, $.paired_statement),
       alias($.filter_statement, $.paired_statement),
-      $.unpaired_statement,
+      $.unpaired_statement
-      $.detatched_end_statement
     ),
 
     paired_statement: $ => {
@@ -73,13 +93,13 @@ module.exports = grammar({
       ];
 
       return choice(...tag_names.map((tag_name) => seq(
-        "{%", $._ws, alias(tag_name + " ", $.tag_name), optional($._ws), repeat($._attribute), "%}",
+        "{%", alias(tag_name, $.tag_name), repeat($._attribute), "%}",
         repeat($._node),
-        "{%", $._ws, "end", alias(tag_name + " ", $.tag_name), optional($._ws), repeat($._attribute), alias("%}", $.end_paired_statement))));
+        "{%", alias("end" + tag_name, $.tag_name), repeat($._attribute), alias("%}", $.end_paired_statement))));
     },
 
     if_statement: $ => seq(
-      "{%", $._ws, alias("if ", $.tag_name), optional($._ws), repeat($._attribute), "%}",
+      "{%", alias("if", $.tag_name), repeat($._attribute), "%}",
       repeat($._node),
       repeat(prec.left(seq(
         alias($.elif_statement, $.branch_statement),
@@ -89,30 +109,28 @@ module.exports = grammar({
         alias($.else_statement, $.branch_statement),
         repeat($._node),
       )),
-      "{%", $._ws, "end", alias("if ", $.tag_name), optional($._ws), alias("%}", $.end_paired_statement)
+      "{%", alias("endif", $.tag_name), alias("%}", $.end_paired_statement)
     ),
-    elif_statement: $ => seq("{%", $._ws, alias("elif ", $.tag_name), optional($._ws), repeat($._attribute), "%}"),
+    elif_statement: $ => seq("{%", alias("elif", $.tag_name), repeat($._attribute), "%}"),
-    else_statement: $ => seq("{%", $._ws, alias("else ", $.tag_name), optional($._ws), "%}"),
+    else_statement: $ => seq("{%", alias("else", $.tag_name), "%}"),
 
     for_statement: $ => seq(
-      "{%", $._ws, alias("for ", $.tag_name), optional($._ws), repeat($._attribute), "%}",
+      "{%", alias("for", $.tag_name), repeat($._attribute), "%}",
       repeat($._node),
       optional(seq(
         alias($.empty_statement, $.branch_statement),
         repeat($._node),
       )),
-      "{%", $._ws, "end", alias("for ", $.tag_name), optional($._ws), alias("%}", $.end_paired_statement)
+      "{%", alias("endfor", $.tag_name), alias("%}", $.end_paired_statement)
     ),
-    empty_statement: $ => seq("{%", $._ws, alias("empty ", $.tag_name), optional($._ws), repeat($._attribute), "%}"),
+    empty_statement: $ => seq("{%", alias("empty", $.tag_name), repeat($._attribute), "%}"),
 
     filter_statement: $ => seq(
-      "{%", $._ws, alias("filter ", $.tag_name), optional($._ws), $.filter, repeat(seq("|", $.filter)), $._ws, "%}",
+      "{%", alias("filter", $.tag_name), $.filter, repeat(seq("|", $.filter)), "%}",
       repeat($._node),
-      "{%", $._ws, "end", alias("filter ", $.tag_name), optional($._ws), alias("%}", $.end_paired_statement)
+      "{%", alias("endfilter", $.tag_name), alias("%}", $.end_paired_statement)
     ),
-    
+    unpaired_statement: $ => seq("{%", alias($._identifier, $.tag_name), repeat($._attribute), "%}"),
-    unpaired_statement: $ => seq("{%", $._ws, alias($._word, $.tag_name), $._ws, repeat($._attribute), "%}"),
-    detatched_end_statement: $ => seq("{%", $._ws, "end", alias($._word, $.tag_name), $._ws, repeat($._attribute), "%}"),
 
     _attribute: $ => seq(
       choice(
@@ -124,11 +142,7 @@ module.exports = grammar({
         $.string,
         $.variable
       ),
-      choice(
+      optional(choice(",", "="))
-        $._ws,
-        seq(optional($._ws), ",", optional($._ws)),
-        seq(optional($._ws), "=", optional($._ws))
-      )
     ),
 
     // Comments
@@ -140,10 +154,10 @@ module.exports = grammar({
     ),
     unpaired_comment: $ => seq("{#", repeat(/.|\s/), repeat(seq(alias($.unpaired_comment, ""), repeat(/.|\s/))), "#}"),
     paired_comment: $ => seq(
-      alias("{%", ""), $._ws, "comment", optional(seq($._ws, $._word)), $._ws, alias("%}", ""),
+      alias("{%", ""), "comment", optional($._identifier), alias("%}", ""),
       repeat(/.|\s/),
       repeat(seq(alias($.paired_comment, ""), repeat(/.|\s/))),
-      alias("{%", ""), $._ws, "endcomment", $._ws, alias("%}", "")
+      alias("{%", ""), "endcomment", alias("%}", "")
     ),
 
     // All other content

package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-htmldjango",
-  "version": "0.1.1",
+  "version": "1.0.0",
   "description": "A tree-sitter grammar for the Django template language",
   "main": "bindings/node",
   "scripts": {

queries/highlights.scm

@@ -12,7 +12,6 @@
 ] @tag
 
 [
- "end"
  (tag_name) 
 ] @function
 

src/node-types.json

@@ -4,11 +4,6 @@
     "named": false,
     "fields": {}
   },
-  {
-    "type": "boolean",
-    "named": true,
-    "fields": {}
-  },
   {
     "type": "branch_statement",
     "named": true,
@@ -52,49 +47,6 @@
       ]
     }
   },
-  {
-    "type": "detatched_end_statement",
-    "named": true,
-    "fields": {},
-    "children": {
-      "multiple": true,
-      "required": true,
-      "types": [
-        {
-          "type": "boolean",
-          "named": true
-        },
-        {
-          "type": "keyword",
-          "named": true
-        },
-        {
-          "type": "keyword_operator",
-          "named": true
-        },
-        {
-          "type": "number",
-          "named": true
-        },
-        {
-          "type": "operator",
-          "named": true
-        },
-        {
-          "type": "string",
-          "named": true
-        },
-        {
-          "type": "tag_name",
-          "named": true
-        },
-        {
-          "type": "variable",
-          "named": true
-        }
-      ]
-    }
-  },
   {
     "type": "filter",
     "named": true,
@@ -124,21 +76,6 @@
     "named": true,
     "fields": {}
   },
-  {
-    "type": "keyword",
-    "named": true,
-    "fields": {}
-  },
-  {
-    "type": "keyword_operator",
-    "named": true,
-    "fields": {}
-  },
-  {
-    "type": "number",
-    "named": true,
-    "fields": {}
-  },
   {
     "type": "operator",
     "named": true,
@@ -169,10 +106,6 @@
           "type": "content",
           "named": true
         },
-        {
-          "type": "detatched_end_statement",
-          "named": true
-        },
         {
           "type": "end_paired_statement",
           "named": true
@@ -243,11 +176,6 @@
       ]
     }
   },
-  {
-    "type": "tag_name",
-    "named": true,
-    "fields": {}
-  },
   {
     "type": "template",
     "named": true,
@@ -260,10 +188,6 @@
           "type": "content",
           "named": true
         },
-        {
-          "type": "detatched_end_statement",
-          "named": true
-        },
         {
           "type": "paired_comment",
           "named": true
@@ -354,19 +278,10 @@
       ]
     }
   },
-  {
-    "type": "variable_name",
-    "named": true,
-    "fields": {}
-  },
   {
     "type": "",
     "named": false
   },
-  {
-    "type": " ",
-    "named": false
-  },
   {
     "type": "!=",
     "named": false
@@ -424,24 +339,8 @@
     "named": false
   },
   {
-    "type": "False",
+    "type": "boolean",
-    "named": false
+    "named": true
-  },
-  {
-    "type": "True",
-    "named": false
-  },
-  {
-    "type": "and",
-    "named": false
-  },
-  {
-    "type": "as",
-    "named": false
-  },
-  {
-    "type": "by",
-    "named": false
   },
   {
     "type": "comment",
@@ -451,10 +350,6 @@
     "type": "content",
     "named": true
   },
-  {
-    "type": "end",
-    "named": false
-  },
   {
     "type": "end_paired_statement",
     "named": true
@@ -464,56 +359,24 @@
     "named": false
   },
   {
-    "type": "from",
+    "type": "keyword",
-    "named": false
+    "named": true
   },
   {
-    "type": "in",
+    "type": "keyword_operator",
-    "named": false
+    "named": true
   },
   {
-    "type": "is",
+    "type": "number",
-    "named": false
+    "named": true
   },
   {
-    "type": "is not",
+    "type": "tag_name",
-    "named": false
+    "named": true
   },
   {
-    "type": "not",
+    "type": "variable_name",
-    "named": false
+    "named": true
-  },
-  {
-    "type": "not in",
-    "named": false
-  },
-  {
-    "type": "off",
-    "named": false
-  },
-  {
-    "type": "on",
-    "named": false
-  },
-  {
-    "type": "only",
-    "named": false
-  },
-  {
-    "type": "or",
-    "named": false
-  },
-  {
-    "type": "random",
-    "named": false
-  },
-  {
-    "type": "silent",
-    "named": false
-  },
-  {
-    "type": "with",
-    "named": false
   },
   {
     "type": "{#",

src/tree_sitter/parser.h

@@ -13,9 +13,8 @@ extern "C" {
 #define ts_builtin_sym_end 0
 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
 
-typedef uint16_t TSStateId;
-
 #ifndef TREE_SITTER_API_H_
+typedef uint16_t TSStateId;
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
@@ -130,9 +129,16 @@ struct TSLanguage {
  *  Lexer Macros
  */
 
+#ifdef _MSC_VER
+#define UNUSED __pragma(warning(suppress : 4101))
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
 #define START_LEXER()           \
   bool result = false;          \
   bool skip = false;            \
+  UNUSED                        \
   bool eof = false;             \
   int32_t lookahead;            \
   goto start;                   \
@@ -166,7 +172,7 @@ struct TSLanguage {
  *  Parse Table Macros
  */
 
-#define SMALL_STATE(id) id - LARGE_STATE_COUNT
+#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
 
 #define STATE(id) id
 
@@ -176,7 +182,7 @@ struct TSLanguage {
   {{                                  \
     .shift = {                        \
       .type = TSParseActionTypeShift, \
-      .state = state_value            \
+      .state = (state_value)          \
     }                                 \
   }}
 
@@ -184,7 +190,7 @@ struct TSLanguage {
   {{                                  \
     .shift = {                        \
       .type = TSParseActionTypeShift, \
-      .state = state_value,           \
+      .state = (state_value),         \
       .repetition = true              \
     }                                 \
   }}

test/corpus/statements.txt

@@ -586,6 +586,31 @@ with
     (end_paired_statement))
   (content))
 
+==================
+keywords
+==================
+
+<html>
+  {% get_word as word %}
+  {% get_word as randomword %}
+</html>
+
+---
+
+(template
+  (content)
+  (unpaired_statement
+    (tag_name)
+    (keyword)
+    (variable
+      (variable_name)))
+  (unpaired_statement
+    (tag_name)
+    (keyword)
+    (variable
+      (variable_name)))
+  (content))
+
 ==================
 custom
 ==================
@@ -612,16 +637,16 @@ custom
   (unpaired_statement
     (tag_name))
   (content)
-  (detatched_end_statement
+  (unpaired_statement
       (tag_name))
   (unpaired_statement
     (tag_name))
   (content)
-  (detatched_end_statement
+  (unpaired_statement
       (tag_name))
   (unpaired_statement
     (tag_name))
   (content)
-  (detatched_end_statement
+  (unpaired_statement
       (tag_name))
   (content))