Initial commit for semantic tokenizer.

mint-lang · gdotdesign · Sep 17, 2023 · Jun 12, 2023 · Jun 13, 2023 · Jun 13, 2023
commit 8ac5b1d8037ab0883a34855e938adee93d7a1386
diff --git a/src/all.cr b/src/all.cr
@@ -57,6 +57,8 @@ require "./documentation_generator/**"
 require "./documentation_generator"
 require "./documentation_server"
 
+require "./semantic_tokenizer"
+
 require "./test_runner/**"
 require "./test_runner"
 

diff --git a/src/ast.cr b/src/ast.cr
@@ -31,9 +31,10 @@ module Mint
                        Js
 
     getter components, modules, records, stores, routes, providers
-    getter suites, enums, comments, nodes, unified_modules
+    getter suites, enums, comments, nodes, unified_modules, keywords
 
-    def initialize(@records = [] of RecordDefinition,
+    def initialize(@keywords = [] of Tuple(Int32, Int32),
+                   @records = [] of RecordDefinition,
                    @unified_modules = [] of Module,
                    @components = [] of Component,
                    @providers = [] of Provider,

diff --git a/src/ast/html_component.cr b/src/ast/html_component.cr
@@ -1,13 +1,14 @@
 module Mint
   class Ast
     class HtmlComponent < Node
-      getter attributes, children, component, comments, ref
+      getter attributes, children, component, comments, ref, closing_tag_position
 
       def initialize(@attributes : Array(HtmlAttribute),
+                     @closing_tag_position : Int32?,
                      @comments : Array(Comment),
                      @children : Array(Node),
-                     @ref : Variable?,
                      @component : TypeId,
+                     @ref : Variable?,
                      @input : Data,
                      @from : Int32,
                      @to : Int32)

diff --git a/src/ast/html_element.cr b/src/ast/html_element.cr
@@ -2,8 +2,10 @@ module Mint
   class Ast
     class HtmlElement < Node
       getter attributes, children, styles, tag, comments, ref
+      getter closing_tag_position
 
       def initialize(@attributes : Array(HtmlAttribute),
+                     @closing_tag_position : Int32?,
                      @comments : Array(Comment),
                      @styles : Array(HtmlStyle),
                      @children : Array(Node),

diff --git a/src/ast/node.cr b/src/ast/node.cr
@@ -71,7 +71,7 @@ module Mint
         source.strip.includes?('\n')
       end
 
-      protected def compute_position(lines, needle) : Position
+      def self.compute_position(lines, needle) : Position
         line_start_pos, line = begin
           left, right = 0, lines.size - 1
           index = pos = 0
@@ -107,19 +107,23 @@ module Mint
         {line, column}
       end
 
-      getter location : Location do
+      def self.compute_location(input : Data, from, to)
         # TODO: avoid creating this array for every (initial) call to `Node#location`
         lines = [0]
-        @input.input.each_char_with_index do |ch, i|
+        input.input.each_char_with_index do |ch, i|
           lines << i + 1 if ch == '\n'
         end
 
         Location.new(
-          filename: @input.file,
+          filename: input.file,
           start: compute_position(lines, from),
           end: compute_position(lines, to),
         )
       end
+
+      getter location : Location do
+        Node.compute_location(input, from, to)
+      end
     end
   end
 end
diff --git a/src/cli.cr b/src/cli.cr
@@ -15,6 +15,7 @@ module Mint
     define_help description: "Mint"
 
     register_sub_command "sandbox-server", type: SandboxServer
+    register_sub_command highlight, type: Highlight
     register_sub_command install, type: Install
     register_sub_command compile, type: Compile
     register_sub_command version, type: Version

diff --git a/src/commands/highlight.cr b/src/commands/highlight.cr
@@ -0,0 +1,84 @@
+module Mint
+  class Cli < Admiral::Command
+    class Highlight < Admiral::Command
+      include Command
+
+      define_help description: "Returns the syntax highlighted version of the given file as HTML"
+
+      define_argument path,
+        description: "The path to the file"
+
+      def run
+        if path = arguments.path
+          ast =
+            Parser.parse(path)
+
+          tokenizer = SemanticTokenizer.new
+          tokenizer.tokenize(ast)
+
+          parts = [] of String | Tuple(String, SemanticTokenizer::TokenType)
+          contents = File.read(path)
+          position = 0
+
+          tokenizer.tokens.sort_by(&.from).each do |token|
+            if token.from > position
+              parts << contents[position, token.from - position]
+            end
+
+            parts << {contents[token.from, token.to - token.from], token.type}
+            position = token.to
+          end
+
+          if position < contents.size
+            parts << contents[position, contents.size]
+          end
+
+          result = parts.reduce("") do |memo, item|
+            memo + case item
+            in String
+              item
+            in Tuple(String, SemanticTokenizer::TokenType)
+              case item[1]
+              in SemanticTokenizer::TokenType::Type
+                item[0].colorize(:yellow)
+              in SemanticTokenizer::TokenType::TypeParameter
+                item[0].colorize(:light_yellow)
+              in SemanticTokenizer::TokenType::Variable
+                item[0].colorize(:dark_gray)
+              in SemanticTokenizer::TokenType::Class
+                item[0].colorize(:blue)
+              in SemanticTokenizer::TokenType::Struct
+                item[0].colorize.fore(:white).back(:red)
+              in SemanticTokenizer::TokenType::Namespace
+                item[0].colorize(:light_blue)
+              in SemanticTokenizer::TokenType::Function
+                item[0].colorize.fore(:white).back(:red)
+              in SemanticTokenizer::TokenType::Keyword
+                item[0].colorize(:magenta)
+              in SemanticTokenizer::TokenType::Property
+                item[0].colorize(:dark_gray).mode(:underline)
+              in SemanticTokenizer::TokenType::Comment
+                item[0].colorize(:light_gray)
+              in SemanticTokenizer::TokenType::Enum
+                item[0].colorize.fore(:white).back(:red)
+              in SemanticTokenizer::TokenType::EnumMember
+                item[0].colorize.fore(:white).back(:red)
+              in SemanticTokenizer::TokenType::String
+                item[0].colorize(:green)
+              in SemanticTokenizer::TokenType::Number
+                item[0].colorize.fore(:white).back(:red)
+              in SemanticTokenizer::TokenType::Regexp
+                item[0].colorize.fore(:white).back(:red)
+              in SemanticTokenizer::TokenType::Operator
+                item[0].colorize.fore(:white).back(:red)
+              end.to_s
+              # %(<span class="#{html_class}">#{item[0]}</span>)
+            end
+          end
+
+          print result
+        end
+      end
+    end
+  end
+end
diff --git a/src/parser.cr b/src/parser.cr
@@ -167,8 +167,8 @@ module Mint
     # Consuming keywords
     # ----------------------------------------------------------------------------
 
-    def keyword!(word, error) : Bool
-      keyword(word) || raise error
+    def keyword!(word, error, save : Bool = false) : Bool
+      keyword(word, save) || raise error
     end
 
     def keyword_ahead?(word) : Bool
@@ -178,8 +178,12 @@ module Mint
       true
     end
 
-    def keyword(word) : Bool
+    def keyword(word, save : Bool = false) : Bool
       if keyword_ahead?(word)
+        if save
+          @ast.keywords << {position, position + word.size}
+        end
+
         @position += word.size
         true
       else

diff --git a/src/parsers/array_literal.cr b/src/parsers/array_literal.cr
@@ -18,7 +18,7 @@ module Mint
 
         type = start do
           whitespace
-          next unless keyword "of"
+          next unless keyword("of", true)
           whitespace
           type_or_type_variable! ArrayLiteralExpectedTypeOrVariable
         end

diff --git a/src/parsers/case.cr b/src/parsers/case.cr
@@ -8,14 +8,14 @@ module Mint
 
     def case_expression(for_css : Bool = false) : Ast::Case?
       start do |start_position|
-        next unless keyword "case"
+        next unless keyword("case", true)
 
         whitespace
 
         parens = char! '('
 
         whitespace
-        await = keyword "await"
+        await = keyword("await", true)
 
         whitespace
         condition = expression! CaseExpectedCondition

diff --git a/src/parsers/component.cr b/src/parsers/component.cr
@@ -9,10 +9,10 @@ module Mint
       start do |start_position|
         comment = self.comment
 
-        global = keyword "global"
+        global = keyword("global", true)
         whitespace
 
-        next unless keyword "component"
+        next unless keyword("component", true)
         whitespace
 
         name = type_id! ComponentExpectedName

diff --git a/src/parsers/connect.cr b/src/parsers/connect.cr
@@ -8,13 +8,13 @@ module Mint
 
     def connect : Ast::Connect?
       start do |start_position|
-        next unless keyword "connect"
+        next unless keyword("connect", true)
         whitespace
 
         store = type_id! ConnectExpectedType
         whitespace
 
-        keyword! "exposing", ConnectExpectedExposing
+        keyword!("exposing", ConnectExpectedExposing, true)
 
         keys = block(
           opening_bracket: ConnectExpectedOpeningBracket,

diff --git a/src/parsers/connect_variable.cr b/src/parsers/connect_variable.cr
@@ -10,7 +10,7 @@ module Mint
 
         whitespace
 
-        if keyword "as"
+        if keyword("as", true)
           whitespace
           name = variable! ConnectVariableExpectedAs
         end

diff --git a/src/parsers/constant.cr b/src/parsers/constant.cr
@@ -9,7 +9,7 @@ module Mint
         comment = self.comment
         whitespace
 
-        next unless keyword "const"
+        next unless keyword("const", true)
         whitespace
 
         name = variable_constant!

diff --git a/src/parsers/css_font_face.cr b/src/parsers/css_font_face.cr
@@ -5,7 +5,7 @@ module Mint
 
     def css_font_face : Ast::CssFontFace?
       start do |start_position|
-        next unless keyword "@font-face"
+        next unless keyword("@font-face", true)
 
         definitions = block(
           opening_bracket: CssFontFaceExpectedOpeningBracket,

diff --git a/src/parsers/css_keyframes.cr b/src/parsers/css_keyframes.cr
@@ -6,7 +6,7 @@ module Mint
 
     def css_keyframes : Ast::CssKeyframes?
       start do |start_position|
-        next unless keyword "@keyframes"
+        next unless keyword("@keyframes", true)
 
         whitespace
 

diff --git a/src/parsers/decode.cr b/src/parsers/decode.cr
@@ -6,15 +6,15 @@ module Mint
 
     def decode : Ast::Decode?
       start do |start_position|
-        next unless keyword "decode"
+        next unless keyword("decode", true)
         next unless whitespace?
         whitespace
 
-        unless keyword "as"
+        unless keyword("as", true)
           expression = expression! DecodeExpectedExpression
 
           whitespace
-          keyword! "as", DecodeExpectedAs
+          keyword!("as", DecodeExpectedAs, true)
         end
 
         whitespace

diff --git a/src/parsers/directives/asset.cr b/src/parsers/directives/asset.cr
@@ -6,7 +6,7 @@ module Mint
 
     def asset_directive : Ast::Directives::Asset?
       start do |start_position|
-        next unless keyword "@asset"
+        next unless keyword("@asset", true)
 
         char '(', AssetDirectiveExpectedOpeningParentheses
         whitespace

diff --git a/src/parsers/directives/documentation.cr b/src/parsers/directives/documentation.cr
@@ -6,7 +6,7 @@ module Mint
 
     def documentation_directive : Ast::Directives::Documentation?
       start do |start_position|
-        next unless keyword "@documentation"
+        next unless keyword("@documentation", true)
 
         char '(', DocumentationDirectiveExpectedOpeningParentheses
         whitespace

diff --git a/src/parsers/directives/format.cr b/src/parsers/directives/format.cr
@@ -6,7 +6,7 @@ module Mint
 
     def format_directive : Ast::Directives::Format?
       start do |start_position|
-        next unless keyword "@format"
+        next unless keyword("@format", true)
 
         content =
           code_block(

diff --git a/src/parsers/directives/inline.cr b/src/parsers/directives/inline.cr
@@ -6,7 +6,7 @@ module Mint
 
     def inline_directive : Ast::Directives::Inline?
       start do |start_position|
-        next unless keyword "@inline"
+        next unless keyword("@inline", true)
 
         char '(', InlineDirectiveExpectedOpeningParentheses
         whitespace

diff --git a/src/parsers/directives/svg.cr b/src/parsers/directives/svg.cr
@@ -6,7 +6,7 @@ module Mint
 
     def svg_directive : Ast::Directives::Svg?
       start do |start_position|
-        next unless keyword "@svg"
+        next unless keyword("@svg", true)
 
         char '(', SvgDirectiveExpectedOpeningParentheses
         whitespace

diff --git a/src/parsers/encode.cr b/src/parsers/encode.cr
@@ -4,7 +4,7 @@ module Mint
 
     def encode : Ast::Encode?
       start do |start_position|
-        next unless keyword "encode"
+        next unless keyword("encode", true)
         next unless whitespace?
         whitespace
 

diff --git a/src/parsers/enum.cr b/src/parsers/enum.cr
@@ -9,7 +9,7 @@ module Mint
       start do |start_position|
         comment = self.comment
 
-        next unless keyword "enum"
+        next unless keyword("enum", true)
         whitespace
 
         name = type_id! EnumExpectedName

diff --git a/src/parsers/for.cr b/src/parsers/for.cr
@@ -9,7 +9,7 @@ module Mint
 
     def for_expression : Ast::For?
       start do |start_position|
-        next unless keyword "for"
+        next unless keyword("for", true)
         next unless whitespace?
         whitespace
 
@@ -22,7 +22,7 @@ module Mint
         ) { variable }
 
         whitespace
-        keyword! "of", ForExpectedOf
+        keyword!("of", ForExpectedOf, true)
         whitespace
 
         subject = expression! ForExpectedSubject