Simplify semantic tokenizer a bit.

mint-lang · gdotdesign · Sep 17, 2023 · Jun 12, 2023 · Jun 13, 2023 · Jun 13, 2023
commit 60f1bb51614ca1074afcce15052e1cdb84314fea
diff --git a/src/commands/highlight.cr b/src/commands/highlight.cr
@@ -66,7 +66,7 @@ module Mint
               in SemanticTokenizer::TokenType::String
                 item[0].colorize(:green)
               in SemanticTokenizer::TokenType::Number
-                item[0].colorize.fore(:white).back(:red)
+                item[0].colorize(:red)
               in SemanticTokenizer::TokenType::Regexp
                 item[0].colorize.fore(:white).back(:red)
               in SemanticTokenizer::TokenType::Operator

diff --git a/src/ls/definition/type_id.cr b/src/ls/definition/type_id.cr
@@ -0,0 +1,30 @@
+module Mint
+  module LS
+    class Definition < LSP::RequestMessage
+      def definition(node : Ast::TypeId, server : Server, workspace : Workspace, stack : Array(Ast::Node))
+        found =
+          workspace.ast.enums.find(&.name.value.==(node.value)) ||
+            workspace.ast.records.find(&.name.value.==(node.value)) ||
+            workspace.ast.stores.find(&.name.value.==(node.value)) ||
+            find_component(workspace, node.value)
+
+        if found.nil? && (next_node = stack[1])
+          definition(next_node, server, workspace, stack)
+        else
+          return if Core.ast.nodes.includes?(found)
+
+          case found
+          when Ast::Store
+            location_link server, node, found.name, found
+          when Ast::Enum
+            location_link server, node, found.name, found
+          when Ast::Component
+            location_link server, node, found.name, found
+          when Ast::RecordDefinition
+            location_link server, node, found.name, found
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/src/parser.cr b/src/parser.cr
@@ -26,13 +26,19 @@ module Mint
       begin
         node = yield position
         @position = start_position unless node
+        clear_nodes(start_position) unless node
         node
       rescue error : Error
         @position = start_position
+        clear_nodes(start_position)
         raise error
       end
     end
 
+    def clear_nodes(from_position)
+      ast.nodes.reject! { |node| node.from >= from_position }
+    end
+
     def step
       @position += 1
     end

diff --git a/src/parsers/get.cr b/src/parsers/get.cr
@@ -11,7 +11,7 @@ module Mint
       start do |start_position|
         comment = self.comment
 
-        next unless keyword "get"
+        next unless keyword("get", true)
         whitespace
 
         name = variable! GetExpectedName, track: false

diff --git a/src/parsers/type_id.cr b/src/parsers/type_id.cr
@@ -1,6 +1,6 @@
 module Mint
   class Parser
-    def type_id!(error : SyntaxError.class) : Ast::TypeId
+    def type_id!(error : SyntaxError.class, track : Bool = true) : Ast::TypeId
       start do |start_position|
         value = gather do
           char(error, &.ascii_uppercase?)
@@ -10,19 +10,21 @@ module Mint
         raise error unless value
 
         if char! '.'
-          other = type_id! error
+          other = type_id!(error, false)
           value += ".#{other.value}"
         end
 
         Ast::TypeId.new(
           from: start_position,
           value: value,
           to: position,
-          input: data)
+          input: data).tap do |node|
+          self << node if track
+        end
       end
     end
 
-    def type_id : Ast::TypeId?
+    def type_id(track : Bool = true) : Ast::TypeId?
       start do |start_position|
         value = gather do
           return unless char.ascii_uppercase?
@@ -36,7 +38,7 @@ module Mint
           if char == '.'
             other = start do
               step
-              next_part = type_id
+              next_part = type_id(false)
               next unless next_part
               next_part
             end
@@ -51,13 +53,15 @@ module Mint
           from: start_position,
           value: value,
           to: position,
-          input: data)
+          input: data).tap do |node|
+          self << node if track
+        end
       end
     end
 
-    def type_id(error : SyntaxError.class) : Ast::TypeId?
+    def type_id(error : SyntaxError.class, track : Bool = true) : Ast::TypeId?
       return unless char.ascii_uppercase?
-      type_id! error
+      type_id! error, track
     end
   end
 end
diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr
@@ -35,63 +35,18 @@ module Mint
         add(from, to, TokenType::Keyword)
       end
 
-      tokenize(
-        ast.records +
-        ast.providers +
-        ast.components +
-        ast.modules +
-        ast.routes +
-        ast.stores +
-        ast.suites +
-        ast.enums +
-        ast.comments)
-    end
-
-    def tokenize(node : Ast::Component)
-      add(node.name, TokenType::Type)
-
-      tokenize(node.comment)
-
-      tokenize(
-        node.properties +
-        node.functions +
-        node.constants +
-        node.connects +
-        node.comments +
-        node.styles +
-        node.states +
-        node.gets +
-        node.uses)
-    end
-
-    def tokenize(node : Ast::Style)
-      add(node.name, TokenType::Variable)
-      tokenize(node.arguments)
-      tokenize(node.body)
+      tokenize(ast.nodes)
     end
 
     def tokenize(node : Ast::CssDefinition)
       add(node.from, node.from + node.name.size, TokenType::Property)
-      tokenize(node.value.select(Ast::Node))
     end
 
-    def tokenize(node : Ast::Function)
-      add(node.name, TokenType::Variable)
-
-      tokenize(node.arguments)
-      tokenize(node.comment)
-      tokenize(node.type)
-      tokenize(node.body)
-    end
-
-    def tokenize(node : Ast::Type)
-      add(node.name, TokenType::Type)
-
-      tokenize(node.parameters)
-    end
-
-    def tokenize(node : Ast::Block)
-      tokenize(node.statements)
+    def tokenize(node : Ast::ArrayAccess)
+      case index = node.index
+      when Int64
+        add(node.from + 1, node.from + 1 + index.to_s.size, TokenType::Number)
+      end
     end
 
     def tokenize(node : Ast::HtmlElement)
@@ -100,50 +55,36 @@ module Mint
       end
 
       add(node.tag, TokenType::Namespace)
-      add(node.ref, TokenType::Variable)
-
-      tokenize(node.attributes)
-      tokenize(node.comments)
-      tokenize(node.children)
-      tokenize(node.styles)
     end
 
     def tokenize(node : Ast::HtmlComponent)
       node.closing_tag_position.try do |position|
         add(position, position + node.component.value.size, TokenType::Type)
       end
-
-      add(node.component, TokenType::Type)
-      add(node.ref, TokenType::Variable)
-
-      tokenize(node.attributes)
-      tokenize(node.comments)
-      tokenize(node.children)
     end
 
-    def tokenize(node : Ast::HtmlAttribute)
-      add(node.name, TokenType::Variable)
-
-      tokenize(node.value)
+    def tokenize(node : Ast::StringLiteral)
+      add(node, TokenType::String)
     end
 
-    def tokenize(node : Ast::HtmlStyle)
-      add(node.name, TokenType::Variable)
+    def tokenize(node : Ast::BoolLiteral)
+      add(node, TokenType::Keyword)
+    end
 
-      tokenize(node.arguments)
+    def tokenize(node : Ast::NumberLiteral)
+      add(node, TokenType::Number)
     end
 
-    def tokenize(node : Ast::StringLiteral)
-      add(node, TokenType::String)
+    def tokenize(node : Ast::Comment)
+      add(node, TokenType::Comment)
     end
 
-    def tokenize(node : Ast::Statement)
-      tokenize(node.expression)
-      tokenize(node.target)
+    def tokenize(node : Ast::Variable)
+      add(node, TokenType::Variable)
     end
 
-    def tokenize(node : Ast::Comment)
-      add(node, TokenType::Comment)
+    def tokenize(node : Ast::TypeId)
+      add(node, TokenType::Type)
     end
 
     def add(from : Int32, to : Int32, type : TokenType)
@@ -161,19 +102,7 @@ module Mint
       nodes.each { |node| tokenize(node) }
     end
 
-    def tokenize(node : Nil)
-    end
-
-    def tokenize(node : Ast::Node)
-      puts "Tokenizer is not implemented for: #{node.class}"
+    def tokenize(node : Ast::Node?)
     end
   end
 end
-
-{% for subclass in Mint::Ast::Node.subclasses %}
-  {% puts "Missing implementation of tokenize for #{subclass.name}" unless Mint::SemanticTokenizer.methods.any? do |method|
-                                                                             if method.name == "tokenize"
-                                                                               method.args[0].restriction.id == subclass.name.gsub(/Mint::/, "")
-                                                                             end
-                                                                           end %}
-{% end %}