Cleanup semantic tokenizer class.

mint-lang · gdotdesign · Sep 17, 2023 · Jun 12, 2023 · Jun 13, 2023 · Jun 13, 2023
commit 1c14a5ae2fdca2764298be576db51f656d2a0126
diff --git a/src/ast.cr b/src/ast.cr
@@ -32,8 +32,10 @@ module Mint
 
     getter components, modules, records, stores, routes, providers
     getter suites, enums, comments, nodes, unified_modules, keywords
+    getter operators
 
-    def initialize(@keywords = [] of Tuple(Int32, Int32),
+    def initialize(@operators = [] of Tuple(Int32, Int32),
+                   @keywords = [] of Tuple(Int32, Int32),
                    @records = [] of RecordDefinition,
                    @unified_modules = [] of Module,
                    @components = [] of Component,

diff --git a/src/commands/highlight.cr b/src/commands/highlight.cr
@@ -45,24 +45,14 @@ module Mint
                 item[0].colorize(:light_yellow)
               in SemanticTokenizer::TokenType::Variable
                 item[0].colorize(:dark_gray)
-              in SemanticTokenizer::TokenType::Class
-                item[0].colorize(:blue)
-              in SemanticTokenizer::TokenType::Struct
-                item[0].colorize.fore(:white).back(:red)
               in SemanticTokenizer::TokenType::Namespace
                 item[0].colorize(:light_blue)
-              in SemanticTokenizer::TokenType::Function
-                item[0].colorize.fore(:white).back(:red)
               in SemanticTokenizer::TokenType::Keyword
                 item[0].colorize(:magenta)
               in SemanticTokenizer::TokenType::Property
                 item[0].colorize(:dark_gray).mode(:underline)
               in SemanticTokenizer::TokenType::Comment
                 item[0].colorize(:light_gray)
-              in SemanticTokenizer::TokenType::Enum
-                item[0].colorize.fore(:white).back(:red)
-              in SemanticTokenizer::TokenType::EnumMember
-                item[0].colorize.fore(:white).back(:red)
               in SemanticTokenizer::TokenType::String
                 item[0].colorize(:green)
               in SemanticTokenizer::TokenType::Number

diff --git a/src/ls/semantic_tokens.cr b/src/ls/semantic_tokens.cr
@@ -11,14 +11,14 @@ module Mint
         ast =
           Parser.parse(uri.path.to_s)
 
+        # This is used later on to convert the line/column of each token
+        input =
+          ast.nodes.first.input
         tokenizer = SemanticTokenizer.new
         tokenizer.tokenize(ast)
 
         data =
           tokenizer.tokens.sort_by(&.from).compact_map do |token|
-            input =
-              ast.nodes.first.input
-
             location =
               Ast::Node.compute_location(input, token.from, token.to)
 

diff --git a/src/lsp/protocol/semantic_tokens_legend.cr b/src/lsp/protocol/semantic_tokens_legend.cr
@@ -0,0 +1,16 @@
+module LSP
+  struct SemanticTokensLegend
+    include JSON::Serializable
+
+    # The token types a server uses.
+    @[JSON::Field(key: "tokenTypes")]
+    property token_types : Array(String)
+
+    # The token modifiers a server uses.
+    @[JSON::Field(key: "tokenModifiers")]
+    property token_modifiers : Array(String)
+
+    def initialize(@token_types, @token_modifiers)
+    end
+  end
+end
diff --git a/src/lsp/protocol/semantic_tokens_options.cr b/src/lsp/protocol/semantic_tokens_options.cr
@@ -1,19 +1,4 @@
 module LSP
-  struct SemanticTokensLegend
-    include JSON::Serializable
-
-    # The token types a server uses.
-    @[JSON::Field(key: "tokenTypes")]
-    property token_types : Array(String)
-
-    # The token modifiers a server uses.
-    @[JSON::Field(key: "tokenModifiers")]
-    property token_modifiers : Array(String)
-
-    def initialize(@token_types, @token_modifiers)
-    end
-  end
-
   struct SemanticTokensOptions
     include JSON::Serializable
 

diff --git a/src/parsers/operation.cr b/src/parsers/operation.cr
@@ -30,9 +30,11 @@ module Mint
     def operator : String?
       start do
         whitespace
+        saved_position = position
         operator = OPERATORS.keys.find { |item| keyword item }
         next unless operator
         next unless whitespace?
+        ast.operators << {saved_position, saved_position + operator.size}
         whitespace
         operator
       end

diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr
@@ -1,55 +1,79 @@
 module Mint
   class SemanticTokenizer
+    # This is a subset of the LSPs SemanticTokenTypes enum.
     enum TokenType
-      Type
       TypeParameter
+      Type
 
-      Variable
-
-      Class     # Component
-      Struct    # Record
-      Namespace # HTML Tags
-      Function
-      Keyword
+      Namespace
       Property
+      Keyword
       Comment
 
-      Enum
-      EnumMember
-
+      Variable
+      Operator
       String
       Number
       Regexp
-      Operator
     end
 
+    # This represents which token types are used for which node.
+    TOKEN_MAP = {
+      Ast::TypeVariable  => TokenType::TypeParameter,
+      Ast::Variable      => TokenType::Variable,
+      Ast::BoolLiteral   => TokenType::Keyword,
+      Ast::Comment       => TokenType::Comment,
+      Ast::StringLiteral => TokenType::String,
+      Ast::NumberLiteral => TokenType::Number,
+      Ast::TypeId        => TokenType::Type,
+    }
+
+    # Represents a semantic token using the positions of the token instead
+    # of line / column (for the LSP it is converted to line /column).
     record Token,
       type : TokenType,
       from : Int32,
       to : Int32
 
+    # We keep a cache of all tokenized nodes to avoid duplications
+    getter cache : Set(Ast::Node) = Set(Ast::Node).new
+
+    # This is where the resulting tokens are stored.
     getter tokens : Array(Token) = [] of Token
 
     def tokenize(ast : Ast)
-      ast.keywords.each do |(from, to)|
-        add(from, to, TokenType::Keyword)
-      end
+      # We add the operators and keywords directly from the AST
+      ast.operators.each { |(from, to)| add(from, to, TokenType::Operator) }
+      ast.keywords.each { |(from, to)| add(from, to, TokenType::Keyword) }
 
       tokenize(ast.nodes)
     end
 
+    def tokenize(nodes : Array(Ast::Node))
+      nodes.each { |node| tokenize(node) }
+    end
+
+    def tokenize(node : Ast::Node?)
+      if type = TOKEN_MAP[node.class]?
+        add(node, type)
+      end
+    end
+
     def tokenize(node : Ast::CssDefinition)
       add(node.from, node.from + node.name.size, TokenType::Property)
     end
 
     def tokenize(node : Ast::ArrayAccess)
+      # TODO: The index should be parsed as a number literal when
+      #       implemented remove this
       case index = node.index
       when Int64
         add(node.from + 1, node.from + 1 + index.to_s.size, TokenType::Number)
       end
     end
 
     def tokenize(node : Ast::HtmlElement)
+      # The closing tag is not saved only the position to it.
       node.closing_tag_position.try do |position|
         add(position, position + node.tag.value.size, TokenType::Namespace)
       end
@@ -63,46 +87,17 @@ module Mint
       end
     end
 
-    def tokenize(node : Ast::StringLiteral)
-      add(node, TokenType::String)
-    end
-
-    def tokenize(node : Ast::BoolLiteral)
-      add(node, TokenType::Keyword)
-    end
-
-    def tokenize(node : Ast::NumberLiteral)
-      add(node, TokenType::Number)
-    end
-
-    def tokenize(node : Ast::Comment)
-      add(node, TokenType::Comment)
-    end
-
-    def tokenize(node : Ast::Variable)
-      add(node, TokenType::Variable)
-    end
-
-    def tokenize(node : Ast::TypeId)
-      add(node, TokenType::Type)
-    end
-
     def add(from : Int32, to : Int32, type : TokenType)
       tokens << Token.new(
         type: type,
         from: from,
         to: to)
     end
 
-    def add(node : Ast::Node | Nil, type : TokenType)
-      add(node.from, node.to, type) if node
-    end
-
-    def tokenize(nodes : Array(Ast::Node))
-      nodes.each { |node| tokenize(node) }
-    end
-
-    def tokenize(node : Ast::Node?)
+    def add(node : Ast::Node, type : TokenType)
+      return if cache.includes?(node)
+      add(node.from, node.to, type)
+      cache.add(node)
     end
   end
 end