Add feature flag to turn on the new Roslyn tokenizer (#11185)

This makes the feature work in VS and VS Code (with corresponding VS Code change to pass the right flag, coming once this is merged) VS Code PR: dotnet/vscode-csharp#7748 /cc @danroth27, @leslierichardson95, @333fred, @chsienki and anyone else who wants, for wording: VS Code: https://github.com/dotnet/vscode-csharp/pull/7748/files#diff-9b29d8b47a74097a04da353b72a8632d485d658282457b97b4caf26575a42dd0R131 VS: https://github.com/dotnet/razor/pull/11185/files#diff-18b9b146f4d27a6a5b6248ece31b30570d84c2a409b25bf5b660de86e608fa4cR80
dotnet · Nov 12, 2024 · f3aa647 · f3aa647
2 parents aeebb01 + 4b7e93d
commit f3aa647
Show file tree

Hide file tree

Showing 13 changed files with 41 additions and 2 deletions.
diff --git a/...azor/src/Microsoft.AspNetCore.Razor.LanguageServer/DefaultLanguageServerFeatureOptions.cs b/...azor/src/Microsoft.AspNetCore.Razor.LanguageServer/DefaultLanguageServerFeatureOptions.cs
@@ -38,4 +38,6 @@ internal class DefaultLanguageServerFeatureOptions : LanguageServerFeatureOption
     public override bool DisableRazorLanguageServer => false;
 
     public override bool ForceRuntimeCodeGeneration => false;
+
+    public override bool UseRoslynTokenizer => false;
 }
diff --git a/...osoft.AspNetCore.Razor.LanguageServer/Hosting/ConfigurableLanguageServerFeatureOptions.cs b/...osoft.AspNetCore.Razor.LanguageServer/Hosting/ConfigurableLanguageServerFeatureOptions.cs
@@ -23,6 +23,7 @@ internal class ConfigurableLanguageServerFeatureOptions : LanguageServerFeatureO
     private readonly bool? _useRazorCohostServer;
     private readonly bool? _disableRazorLanguageServer;
     private readonly bool? _forceRuntimeCodeGeneration;
+    private readonly bool? _useRoslynTokenizer;
 
     public override bool SupportsFileManipulation => _supportsFileManipulation ?? _defaults.SupportsFileManipulation;
     public override string CSharpVirtualDocumentSuffix => _csharpVirtualDocumentSuffix ?? DefaultLanguageServerFeatureOptions.DefaultCSharpVirtualDocumentSuffix;
@@ -37,6 +38,7 @@ internal class ConfigurableLanguageServerFeatureOptions : LanguageServerFeatureO
     public override bool UseRazorCohostServer => _useRazorCohostServer ?? _defaults.UseRazorCohostServer;
     public override bool DisableRazorLanguageServer => _disableRazorLanguageServer ?? _defaults.DisableRazorLanguageServer;
     public override bool ForceRuntimeCodeGeneration => _forceRuntimeCodeGeneration ?? _defaults.ForceRuntimeCodeGeneration;
+    public override bool UseRoslynTokenizer => _useRoslynTokenizer ?? _defaults.UseRoslynTokenizer;
 
     public ConfigurableLanguageServerFeatureOptions(string[] args)
     {
@@ -60,6 +62,7 @@ public ConfigurableLanguageServerFeatureOptions(string[] args)
             TryProcessBoolOption(nameof(UseRazorCohostServer), ref _useRazorCohostServer, option, args, i);
             TryProcessBoolOption(nameof(DisableRazorLanguageServer), ref _disableRazorLanguageServer, option, args, i);
             TryProcessBoolOption(nameof(ForceRuntimeCodeGeneration), ref _forceRuntimeCodeGeneration, option, args, i);
+            TryProcessBoolOption(nameof(UseRoslynTokenizer), ref _useRoslynTokenizer, option, args, i);
         }
     }
 

diff --git a/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/LanguageServerFeatureOptions.cs b/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/LanguageServerFeatureOptions.cs
@@ -36,7 +36,9 @@ internal abstract class LanguageServerFeatureOptions
     public abstract bool DisableRazorLanguageServer { get; }
 
     /// <summary>
-    /// When enabled, design time code will not be generated. All tooling will be using runtime code generation.
+    /// When enabled, design time code will not be generated. All tooling, except formatting, will be using runtime code generation.
     /// </summary>
     public abstract bool ForceRuntimeCodeGeneration { get; }
+
+    public abstract bool UseRoslynTokenizer { get; }
 }
diff --git a/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/ProjectSystem/ProjectState.cs b/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/ProjectSystem/ProjectState.cs
@@ -14,6 +14,7 @@
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.Razor.Workspaces;
 using Microsoft.CodeAnalysis.Text;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 
 namespace Microsoft.CodeAnalysis.Razor.ProjectSystem;
 
@@ -171,12 +172,14 @@ RazorProjectEngine CreateProjectEngine()
             {
                 var configuration = HostProject.Configuration;
                 var rootDirectoryPath = Path.GetDirectoryName(HostProject.FilePath).AssumeNotNull();
+                var useRoslynTokenizer = LanguageServerFeatureOptions.UseRoslynTokenizer;
 
                 return _projectEngineFactoryProvider.Create(configuration, rootDirectoryPath, builder =>
                 {
                     builder.SetRootNamespace(HostProject.RootNamespace);
                     builder.SetCSharpLanguageVersion(CSharpLanguageVersion);
                     builder.SetSupportLocalizedComponentNames();
+                    builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer, CSharpParseOptions.Default));
                 });
             }
         }

diff --git a/...r/src/Microsoft.CodeAnalysis.Razor.Workspaces/Remote/RemoteClientInitializationOptions.cs b/...r/src/Microsoft.CodeAnalysis.Razor.Workspaces/Remote/RemoteClientInitializationOptions.cs
@@ -33,4 +33,7 @@ internal struct RemoteClientInitializationOptions
 
     [JsonPropertyName("showAllCSharpCodeActions")]
     public required bool ShowAllCSharpCodeActions { get; set; }
+
+    [JsonPropertyName("useRoslynTokenizer")]
+    public required bool UseRoslynTokenizer { get; set; }
 }
diff --git a/.../Microsoft.CodeAnalysis.Remote.Razor/Initialization/RemoteLanguageServerFeatureOptions.cs b/.../Microsoft.CodeAnalysis.Remote.Razor/Initialization/RemoteLanguageServerFeatureOptions.cs
@@ -42,4 +42,6 @@ internal class RemoteLanguageServerFeatureOptions : LanguageServerFeatureOptions
     public override bool DisableRazorLanguageServer => throw new InvalidOperationException("This option has not been synced to OOP.");
 
     public override bool ForceRuntimeCodeGeneration => _options.ForceRuntimeCodeGeneration;
+
+    public override bool UseRoslynTokenizer => _options.UseRoslynTokenizer;
 }
diff --git a/src/Razor/src/Microsoft.CodeAnalysis.Remote.Razor/ProjectSystem/RemoteProjectSnapshot.cs b/src/Razor/src/Microsoft.CodeAnalysis.Remote.Razor/ProjectSystem/RemoteProjectSnapshot.cs
@@ -19,6 +19,7 @@
 using Microsoft.CodeAnalysis.Razor;
 using Microsoft.CodeAnalysis.Razor.Compiler.CSharp;
 using Microsoft.CodeAnalysis.Razor.ProjectSystem;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Microsoft.VisualStudio.Threading;
 
 namespace Microsoft.CodeAnalysis.Remote.Razor.ProjectSystem;
@@ -51,6 +52,7 @@ public RemoteProjectSnapshot(Project project, RemoteSolutionSnapshot solutionSna
         _lazyProjectEngine = new AsyncLazy<RazorProjectEngine>(async () =>
         {
             var configuration = await _lazyConfiguration.GetValueAsync();
+            var useRoslynTokenizer = SolutionSnapshot.SnapshotManager.LanguageServerFeatureOptions.UseRoslynTokenizer;
             return ProjectEngineFactories.DefaultProvider.Create(
                 configuration,
                 rootDirectoryPath: Path.GetDirectoryName(FilePath).AssumeNotNull(),
@@ -59,6 +61,7 @@ public RemoteProjectSnapshot(Project project, RemoteSolutionSnapshot solutionSna
                     builder.SetRootNamespace(RootNamespace);
                     builder.SetCSharpLanguageVersion(CSharpLanguageVersion);
                     builder.SetSupportLocalizedComponentNames();
+                    builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer, CSharpParseOptions.Default));
                 });
         },
         joinableTaskFactory: null);

diff --git a/src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/Remote/RemoteServiceInvoker.cs b/src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/Remote/RemoteServiceInvoker.cs
@@ -141,6 +141,7 @@ private async Task InitializeRemoteClientAsync(RazorRemoteHostClient remoteClien
                     ForceRuntimeCodeGeneration = _languageServerFeatureOptions.ForceRuntimeCodeGeneration,
                     SupportsFileManipulation = _languageServerFeatureOptions.SupportsFileManipulation,
                     ShowAllCSharpCodeActions = _languageServerFeatureOptions.ShowAllCSharpCodeActions,
+                    UseRoslynTokenizer = _languageServerFeatureOptions.UseRoslynTokenizer,
                 };
 
                 _logger.LogDebug($"First OOP call, so initializing OOP service.");

diff --git a/...Microsoft.VisualStudio.LanguageServices.Razor/VisualStudioLanguageServerFeatureOptions.cs b/...Microsoft.VisualStudio.LanguageServices.Razor/VisualStudioLanguageServerFeatureOptions.cs
@@ -19,6 +19,7 @@ internal class VisualStudioLanguageServerFeatureOptions : LanguageServerFeatureO
     private readonly Lazy<bool> _useRazorCohostServer;
     private readonly Lazy<bool> _disableRazorLanguageServer;
     private readonly Lazy<bool> _forceRuntimeCodeGeneration;
+    private readonly Lazy<bool> _useRoslynTokenizer;
 
     [ImportingConstructor]
     public VisualStudioLanguageServerFeatureOptions(ILspEditorFeatureDetector lspEditorFeatureDetector)
@@ -71,12 +72,18 @@ public VisualStudioLanguageServerFeatureOptions(ILspEditorFeatureDetector lspEdi
             var forceRuntimeCodeGeneration = featureFlags.IsFeatureEnabled(WellKnownFeatureFlagNames.ForceRuntimeCodeGeneration, defaultValue: false);
             return forceRuntimeCodeGeneration;
         });
+
+        _useRoslynTokenizer = new Lazy<bool>(() =>
+        {
+            var featureFlags = (IVsFeatureFlags)Package.GetGlobalService(typeof(SVsFeatureFlags));
+            var useRoslynTokenizer = featureFlags.IsFeatureEnabled(WellKnownFeatureFlagNames.UseRoslynTokenizer, defaultValue: false);
+            return useRoslynTokenizer;
+        });
     }
 
     // We don't currently support file creation operations on VS Codespaces or VS Liveshare
     public override bool SupportsFileManipulation => !IsCodespacesOrLiveshare;
 
-
     public override string CSharpVirtualDocumentSuffix => ".ide.g.cs";
 
     public override string HtmlVirtualDocumentSuffix => "__virtual.html";
@@ -103,4 +110,6 @@ public VisualStudioLanguageServerFeatureOptions(ILspEditorFeatureDetector lspEdi
 
     /// <inheritdoc />
     public override bool ForceRuntimeCodeGeneration => _forceRuntimeCodeGeneration.Value;
+
+    public override bool UseRoslynTokenizer => _useRoslynTokenizer.Value;
 }
diff --git a/src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/WellKnownFeatureFlagNames.cs b/src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/WellKnownFeatureFlagNames.cs
@@ -11,4 +11,5 @@ internal static class WellKnownFeatureFlagNames
     public const string UseRazorCohostServer = "Razor.LSP.UseRazorCohostServer";
     public const string DisableRazorLanguageServer = "Razor.LSP.DisableRazorLanguageServer";
     public const string ForceRuntimeCodeGeneration = "Razor.LSP.ForceRuntimeCodeGeneration";
+    public const string UseRoslynTokenizer = "Razor.LSP.UseRoslynTokenizer";
 }
diff --git a/...Microsoft.VisualStudio.RazorExtension/Microsoft.VisualStudio.RazorExtension.Custom.pkgdef b/...Microsoft.VisualStudio.RazorExtension/Microsoft.VisualStudio.RazorExtension.Custom.pkgdef
@@ -76,6 +76,13 @@
 "PreviewPaneChannels"="*"
 "VisibleToInternalUsersOnlyChannels"="*"
 
+[$RootKey$\FeatureFlags\Razor\LSP\UseRoslynTokenizer]
+"Description"="Enables some new C# features, like interpolated and raw strings, in Razor files opened in Visual Studio. This matches using '<features>use-roslyn-tokenizer</feature>' in a project file for command line builds, and may result in inconsistencies if this option and your project files do not match."
+"Value"=dword:00000000
+"Title"="Use the C# tokenizer for Razor files in the IDE (requires restart)"
+"PreviewPaneChannels"="*"
+"VisibleToInternalUsersOnlyChannels"="*"
+
 // CacheTag value should be changed when registration file changes
 // See https://devdiv.visualstudio.com/DevDiv/_wiki/wikis/DevDiv.wiki/39345/Manifest-Build-Deployment-and-Setup-Authoring-In-Depth?anchor=example-pkgdef-key for more infomation
 [$RootKey$\SettingsManifests\{13b72f58-279e-49e0-a56d-296be02f0805}]

diff --git a/...osoft.AspNetCore.Razor.Test.Common.Tooling/Workspaces/TestLanguageServerFeatureOptions.cs b/...osoft.AspNetCore.Razor.Test.Common.Tooling/Workspaces/TestLanguageServerFeatureOptions.cs
@@ -37,4 +37,6 @@ internal class TestLanguageServerFeatureOptions(
     public override bool DisableRazorLanguageServer => false;
 
     public override bool ForceRuntimeCodeGeneration => forceRuntimeCodeGeneration;
+
+    public override bool UseRoslynTokenizer => false;
 }
diff --git a/.../test/Microsoft.VisualStudio.LanguageServices.Razor.Test/Cohost/CohostEndpointTestBase.cs b/.../test/Microsoft.VisualStudio.LanguageServices.Razor.Test/Cohost/CohostEndpointTestBase.cs
@@ -80,6 +80,7 @@ protected override async Task InitializeAsync()
             ForceRuntimeCodeGeneration = false,
             SupportsFileManipulation = true,
             ShowAllCSharpCodeActions = false,
+            UseRoslynTokenizer = false,
         };
         UpdateClientInitializationOptions(c => c);