From a218f2233199c65a6911a376ebcba54daea5af59 Mon Sep 17 00:00:00 2001
From: George Waters <gwatersdev@gmail.com>
Date: Wed, 1 Feb 2023 17:41:10 -0500
Subject: [PATCH] Improve Python lexer (#1919)

* Improve Python lexer

This includes three changes to the Python lexer:

1) Move the `decorator` rule before the `operator` rule.

   Before this change, decorators were never getting recognized. This
   was because if there was ever an `@`, it was always satisfying the
   `operator` rule. This remedies that by first checking for the
   `decorator` pattern and then the operator pattern.

2) Recognize functions and classes when they are called.

   Previously, functions and classes were only recognized when they were
   defined. With this change, they will also be recognized and styled
   when they are called.

3) Don't recognize imported modules as `Namespace`s.

   It is not desirable to highlight imported modules like namespaces.
   With this change, they are simply recognized as the general `Name`
   token.

* Add more python visual samples

* Add python class visual sample

---------

Co-authored-by: Tan Le <tan.le@hey.com>
---
 lib/rouge/lexers/python.rb | 13 ++++++++-----
 spec/visual/samples/python | 19 ++++++++++++++++++-
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/lib/rouge/lexers/python.rb b/lib/rouge/lexers/python.rb
index 89676de483..103f6c4724 100644
--- a/lib/rouge/lexers/python.rb
+++ b/lib/rouge/lexers/python.rb
@@ -40,7 +40,7 @@ def self.builtins
       end
 
       def self.builtins_pseudo
-        @builtins_pseudo ||= %w(self None Ellipsis NotImplemented False True)
+        @builtins_pseudo ||= %w(None Ellipsis NotImplemented False True)
       end
 
       def self.exceptions
@@ -86,6 +86,8 @@ def current_string
         rule %r/\\\n/, Text
         rule %r/\\/, Text
 
+        rule %r/@#{dotted_identifier}/i, Name::Decorator
+
         rule %r/(in|is|and|or|not)\b/, Operator::Word
         rule %r/(<<|>>|\/\/|\*\*)=?/, Operator
         rule %r/[-~+\/*%=<>&^|@]=?|!=/, Operator
@@ -93,13 +95,13 @@ def current_string
         rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(import)/ do
           groups Keyword::Namespace,
                  Text,
-                 Name::Namespace,
+                 Name,
                  Text,
                  Keyword::Namespace
         end
 
         rule %r/(import)(\s+)(#{dotted_identifier})/ do
-          groups Keyword::Namespace, Text, Name::Namespace
+          groups Keyword::Namespace, Text, Name
         end
 
         rule %r/(def)((?:\s|\\\s)+)/ do
@@ -112,6 +114,9 @@ def current_string
           push :classname
         end
 
+        rule %r/([a-z_]\w*)[ \t]*(?=(\(.*\)))/m, Name::Function
+        rule %r/([A-Z_]\w*)[ \t]*(?=(\(.*\)))/m, Name::Class
+
         # TODO: not in python 3
         rule %r/`.*?`/, Str::Backtick
         rule %r/([rfbu]{0,2})('''|"""|['"])/i do |m|
@@ -120,8 +125,6 @@ def current_string
           push :generic_string
         end
 
-        rule %r/@#{dotted_identifier}/i, Name::Decorator
-
         # using negative lookbehind so we don't match property names
         rule %r/(?<!\.)#{identifier}/ do |m|
           if self.class.keywords.include? m[0]
diff --git a/spec/visual/samples/python b/spec/visual/samples/python
index f80878f4cc..e12691c3af 100644
--- a/spec/visual/samples/python
+++ b/spec/visual/samples/python
@@ -150,9 +150,26 @@ x @= y
 f'{hello} world {int(x) + 1}'
 f'{{ {4*10} }}'
 f'result: {value:{width}.{precision}}'
-f'{value!r}
+f'{value!r}'
 
 # Unicode identifiers
 α = 10
 def coöperative(б):
     return f"{б} is Russian"
+
+def __init__(self, input_dim: list, output_dim: int, **kwargs):
+    super(AverageEmbedding, self).__init__(**kwargs)
+    self.input_dim = input_dim
+    self.output_dim = output_dim
+
+@abstractmethod
+def evaluate(self, *args, **kwargs):
+    raise NotImplementedError
+
+def _get_metadata(self, input_samples: list[InputSample]) -> Tuple[str, str, int]:
+    project_name, id = self._get_info()
+
+class Spam:
+    pass
+
+spam = Spam()