Wire up the UseRoslynTokenizer feature properly

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/CSharp/ParseOptionsExtensions.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.CodeAnalysis.Razor.Compiler.CSharp;
+
+internal static class ParseOptionsExtensions
+{
+    public static bool UseRoslynTokenizer(this ParseOptions parseOptions)
+        => parseOptions.Features.TryGetValue("use-roslyn-tokenizer", out var useRoslynTokenizerValue) &&
+           string.Equals(useRoslynTokenizerValue, "true", StringComparison.OrdinalIgnoreCase);
+}

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/RazorConfiguration.cs

@@ -3,6 +3,7 @@
 
 using System.Collections.Immutable;
 using System.Linq;
+using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.Extensions.Internal;
 
 namespace Microsoft.AspNetCore.Razor.Language;
@@ -11,32 +12,50 @@ public sealed record class RazorConfiguration(
     RazorLanguageVersion LanguageVersion,
     string ConfigurationName,
     ImmutableArray<RazorExtension> Extensions,
+    LanguageVersion CSharpLanguageVersion = LanguageVersion.Default,
     bool UseConsolidatedMvcViews = true,
-    bool SuppressAddComponentParameter = false)
+    bool SuppressAddComponentParameter = false,
+    bool UseRoslynTokenizer = false,
+    ImmutableArray<string> PreprocessorSymbols = default)
 {
+    public ImmutableArray<string> PreprocessorSymbols
+    {
+        get;
+        init => field = value.NullToEmpty();
+    } = PreprocessorSymbols.NullToEmpty();
+
     public static readonly RazorConfiguration Default = new(
         RazorLanguageVersion.Latest,
         ConfigurationName: "unnamed",
         Extensions: [],
+        CSharpLanguageVersion: CodeAnalysis.CSharp.LanguageVersion.Default,
         UseConsolidatedMvcViews: true,
-        SuppressAddComponentParameter: false);
+        SuppressAddComponentParameter: false,
+        UseRoslynTokenizer: false,
+        PreprocessorSymbols: []);
 
     public bool Equals(RazorConfiguration? other)
         => other is not null &&
            LanguageVersion == other.LanguageVersion &&
            ConfigurationName == other.ConfigurationName &&
+           CSharpLanguageVersion == other.CSharpLanguageVersion &&
            SuppressAddComponentParameter == other.SuppressAddComponentParameter &&
            UseConsolidatedMvcViews == other.UseConsolidatedMvcViews &&
+           UseRoslynTokenizer == other.UseRoslynTokenizer &&
+           PreprocessorSymbols.SequenceEqual(other.PreprocessorSymbols) &&
            Extensions.SequenceEqual(other.Extensions);
 
     public override int GetHashCode()
     {
         var hash = HashCodeCombiner.Start();
         hash.Add(LanguageVersion);
         hash.Add(ConfigurationName);
+        hash.Add(CSharpLanguageVersion);
         hash.Add(Extensions);
         hash.Add(SuppressAddComponentParameter);
         hash.Add(UseConsolidatedMvcViews);
+        hash.Add(UseRoslynTokenizer);
+        hash.Add(PreprocessorSymbols);
         return hash;
     }
 }

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/SourceGenerators/RazorSourceGenerator.RazorProviders.cs

@@ -56,8 +56,7 @@ public partial class RazorSourceGenerator
             var razorConfiguration = new RazorConfiguration(razorLanguageVersion, configurationName ?? "default", Extensions: [], UseConsolidatedMvcViews: true, SuppressAddComponentParameter: !isComponentParameterSupported);
 
             // We use the new tokenizer only when requested for now.
-            var useRoslynTokenizer = parseOptions.Features.TryGetValue("use-roslyn-tokenizer", out var useRoslynTokenizerValue)
-                                    && string.Equals(useRoslynTokenizerValue, "true", StringComparison.OrdinalIgnoreCase);
+            var useRoslynTokenizer = parseOptions.UseRoslynTokenizer();
 
             var razorSourceGenerationOptions = new RazorSourceGenerationOptions()
             {

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/LanguageServer/RazorLanguageServerBenchmarkBase.cs

@@ -73,8 +73,7 @@ await projectManager.UpdateAsync(
             updater =>
             {
                 updater.AddProject(hostProject);
-                var tagHelpers = CommonResources.LegacyTagHelpers;
-                var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers, CodeAnalysis.CSharp.LanguageVersion.CSharp11);
+                var projectWorkspaceState = ProjectWorkspaceState.Create(CommonResources.LegacyTagHelpers);
                 updater.UpdateProjectWorkspaceState(hostProject.Key, projectWorkspaceState);
                 updater.AddDocument(hostProject.Key, hostDocument, text);
             },

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/DefaultLanguageServerFeatureOptions.cs

@@ -39,7 +39,5 @@ internal class DefaultLanguageServerFeatureOptions : LanguageServerFeatureOption
 
     public override bool ForceRuntimeCodeGeneration => false;
 
-    public override bool UseRoslynTokenizer => false;
-
     public override bool UseNewFormattingEngine => false;
 }

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Hosting/ConfigurableLanguageServerFeatureOptions.cs

@@ -23,7 +23,6 @@ internal class ConfigurableLanguageServerFeatureOptions : LanguageServerFeatureO
     private readonly bool? _useRazorCohostServer;
     private readonly bool? _disableRazorLanguageServer;
     private readonly bool? _forceRuntimeCodeGeneration;
-    private readonly bool? _useRoslynTokenizer;
     private readonly bool? _useNewFormattingEngine;
 
     public override bool SupportsFileManipulation => _supportsFileManipulation ?? _defaults.SupportsFileManipulation;
@@ -39,7 +38,6 @@ internal class ConfigurableLanguageServerFeatureOptions : LanguageServerFeatureO
     public override bool UseRazorCohostServer => _useRazorCohostServer ?? _defaults.UseRazorCohostServer;
     public override bool DisableRazorLanguageServer => _disableRazorLanguageServer ?? _defaults.DisableRazorLanguageServer;
     public override bool ForceRuntimeCodeGeneration => _forceRuntimeCodeGeneration ?? _defaults.ForceRuntimeCodeGeneration;
-    public override bool UseRoslynTokenizer => _useRoslynTokenizer ?? _defaults.UseRoslynTokenizer;
     public override bool UseNewFormattingEngine => _useNewFormattingEngine ?? _defaults.UseNewFormattingEngine;
 
     public ConfigurableLanguageServerFeatureOptions(string[] args)
@@ -64,7 +62,6 @@ public ConfigurableLanguageServerFeatureOptions(string[] args)
             TryProcessBoolOption(nameof(UseRazorCohostServer), ref _useRazorCohostServer, option, args, i);
             TryProcessBoolOption(nameof(DisableRazorLanguageServer), ref _disableRazorLanguageServer, option, args, i);
             TryProcessBoolOption(nameof(ForceRuntimeCodeGeneration), ref _forceRuntimeCodeGeneration, option, args, i);
-            TryProcessBoolOption(nameof(UseRoslynTokenizer), ref _useRoslynTokenizer, option, args, i);
             TryProcessBoolOption(nameof(UseNewFormattingEngine), ref _useNewFormattingEngine, option, args, i);
         }
     }

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/ProjectSystem/RazorProjectService.cs

@@ -364,17 +364,17 @@ private Task AddOrUpdateProjectCoreAsync(
 
                 if (!projectWorkspaceState.Equals(ProjectWorkspaceState.Default))
                 {
-                    _logger.LogInformation($"Updating project '{project.Key}' TagHelpers ({projectWorkspaceState.TagHelpers.Length}) and C# Language Version ({projectWorkspaceState.CSharpLanguageVersion}).");
+                    _logger.LogInformation($"Updating project '{project.Key}' TagHelpers ({projectWorkspaceState.TagHelpers.Length}).");
                 }
 
                 updater.UpdateProjectWorkspaceState(project.Key, projectWorkspaceState);
 
                 var currentConfiguration = project.Configuration;
                 var currentRootNamespace = project.RootNamespace;
-                if (currentConfiguration.ConfigurationName == configuration?.ConfigurationName &&
+                if (currentConfiguration == configuration &&
                     currentRootNamespace == rootNamespace)
                 {
-                    _logger.LogTrace($"Updating project '{project.Key}'. The project is already using configuration '{configuration.ConfigurationName}' and root namespace '{rootNamespace}'.");
+                    _logger.LogTrace($"Updating project '{project.Key}'. The project is already using configuration '{configuration.ConfigurationName}' and root namespace '{rootNamespace}' and C# lang version '{configuration.CSharpLanguageVersion}'.");
                     return;
                 }
 
@@ -383,9 +383,9 @@ private Task AddOrUpdateProjectCoreAsync(
                     configuration = FallbackRazorConfiguration.Latest;
                     _logger.LogInformation($"Updating project '{project.Key}' to use the latest configuration ('{configuration.ConfigurationName}')'.");
                 }
-                else if (currentConfiguration.ConfigurationName != configuration.ConfigurationName)
+                else
                 {
-                    _logger.LogInformation($"Updating project '{project.Key}' to Razor configuration '{configuration.ConfigurationName}' with language version '{configuration.LanguageVersion}'.");
+                    _logger.LogInformation($"Updating project '{project.Key}' to Razor configuration '{configuration.ConfigurationName}' with language version '{configuration.LanguageVersion}' and C# lang version '{configuration.CSharpLanguageVersion}'.");
                 }
 
                 if (currentRootNamespace != rootNamespace)

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/ProjectSystem/ProjectWorkspaceState.cs

@@ -5,52 +5,40 @@
 using System.Collections.Immutable;
 using System.Linq;
 using Microsoft.AspNetCore.Razor.Language;
-using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.Extensions.Internal;
 
 namespace Microsoft.AspNetCore.Razor.ProjectSystem;
 
 internal sealed class ProjectWorkspaceState : IEquatable<ProjectWorkspaceState>
 {
-    public static readonly ProjectWorkspaceState Default = new(ImmutableArray<TagHelperDescriptor>.Empty, LanguageVersion.Default);
+    public static readonly ProjectWorkspaceState Default = new(ImmutableArray<TagHelperDescriptor>.Empty);
 
     public ImmutableArray<TagHelperDescriptor> TagHelpers { get; }
-    public LanguageVersion CSharpLanguageVersion { get; }
 
     private ProjectWorkspaceState(
-        ImmutableArray<TagHelperDescriptor> tagHelpers,
-        LanguageVersion csharpLanguageVersion)
+        ImmutableArray<TagHelperDescriptor> tagHelpers)
     {
         TagHelpers = tagHelpers;
-        CSharpLanguageVersion = csharpLanguageVersion;
     }
 
     public static ProjectWorkspaceState Create(
-        ImmutableArray<TagHelperDescriptor> tagHelpers,
-        LanguageVersion csharpLanguageVersion = LanguageVersion.Default)
-        => tagHelpers.IsEmpty && csharpLanguageVersion == LanguageVersion.Default
+        ImmutableArray<TagHelperDescriptor> tagHelpers)
+        => tagHelpers.IsEmpty
             ? Default
-            : new(tagHelpers, csharpLanguageVersion);
-
-    public static ProjectWorkspaceState Create(LanguageVersion csharpLanguageVersion)
-        => csharpLanguageVersion == LanguageVersion.Default
-            ? Default
-            : new(ImmutableArray<TagHelperDescriptor>.Empty, csharpLanguageVersion);
+            : new(tagHelpers);
 
     public override bool Equals(object? obj)
         => obj is ProjectWorkspaceState other && Equals(other);
 
     public bool Equals(ProjectWorkspaceState? other)
         => other is not null &&
-           TagHelpers.SequenceEqual(other.TagHelpers) &&
-           CSharpLanguageVersion == other.CSharpLanguageVersion;
+           TagHelpers.SequenceEqual(other.TagHelpers);
 
     public override int GetHashCode()
     {
         var hash = HashCodeCombiner.Start();
 
         hash.Add(TagHelpers);
-        hash.Add(CSharpLanguageVersion);
 
         return hash.CombinedHash;
     }

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Serialization/MessagePack/Formatters/ProjectWorkspaceStateFormatter.cs

@@ -8,7 +8,6 @@
 using Microsoft.AspNetCore.Razor.ProjectSystem;
 using Microsoft.AspNetCore.Razor.Serialization.MessagePack.Formatters.TagHelpers;
 using Microsoft.AspNetCore.Razor.Utilities;
-using Microsoft.CodeAnalysis.CSharp;
 
 namespace Microsoft.AspNetCore.Razor.Serialization.MessagePack.Formatters;
 
@@ -22,7 +21,7 @@ private ProjectWorkspaceStateFormatter()
 
     public override ProjectWorkspaceState Deserialize(ref MessagePackReader reader, SerializerCachingOptions options)
     {
-        reader.ReadArrayHeaderAndVerify(3);
+        reader.ReadArrayHeaderAndVerify(2);
 
         var checksums = reader.Deserialize<ImmutableArray<Checksum>>(options);
 
@@ -47,19 +46,17 @@ public override ProjectWorkspaceState Deserialize(ref MessagePackReader reader,
         }
 
         var tagHelpers = builder.DrainToImmutable();
-        var csharpLanguageVersion = (LanguageVersion)reader.ReadInt32();
 
-        return ProjectWorkspaceState.Create(tagHelpers, csharpLanguageVersion);
+        return ProjectWorkspaceState.Create(tagHelpers);
     }
 
     public override void Serialize(ref MessagePackWriter writer, ProjectWorkspaceState value, SerializerCachingOptions options)
     {
-        writer.WriteArrayHeader(3);
+        writer.WriteArrayHeader(2);
 
         var checksums = value.TagHelpers.SelectAsArray(x => x.Checksum);
 
         writer.Serialize(checksums, options);
         writer.Serialize(value.TagHelpers, options);
-        writer.Write((int)value.CSharpLanguageVersion);
     }
 }

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Serialization/MessagePack/Formatters/RazorConfigurationFormatter.cs

@@ -1,14 +1,18 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
+using System.Collections.Immutable;
 using MessagePack;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.PooledObjects;
+using Microsoft.CodeAnalysis.CSharp;
 
 namespace Microsoft.AspNetCore.Razor.Serialization.MessagePack.Formatters;
 
 internal sealed class RazorConfigurationFormatter : ValueFormatter<RazorConfiguration>
 {
+    private const int SerializerPropertyCount = 7;
+
     public static readonly ValueFormatter<RazorConfiguration> Instance = new RazorConfigurationFormatter();
 
     private RazorConfigurationFormatter()
@@ -17,15 +21,18 @@ private RazorConfigurationFormatter()
 
     public override RazorConfiguration Deserialize(ref MessagePackReader reader, SerializerCachingOptions options)
     {
-        // The count is the number of values (2 or 3, depending on what was written) + the number of extensions
+        // The count is the number of values + the number of extensions
         var count = reader.ReadArrayHeader();
 
         var configurationName = CachedStringFormatter.Instance.Deserialize(ref reader, options) ?? string.Empty;
         var languageVersionText = CachedStringFormatter.Instance.Deserialize(ref reader, options) ?? string.Empty;
+        var csharpLanguageVersion = (LanguageVersion)reader.ReadInt32();
         var suppressAddComponentParameter = reader.ReadBoolean();
         var useConsolidatedMvcViews = reader.ReadBoolean();
+        var useRoslynTokenizer = reader.ReadBoolean();
+        var preprocessorSymbols = reader.Deserialize<ImmutableArray<string>>(options);
 
-        count -= 4;
+        count -= SerializerPropertyCount;
 
         using var builder = new PooledArrayBuilder<RazorExtension>();
 
@@ -45,15 +52,18 @@ public override RazorConfiguration Deserialize(ref MessagePackReader reader, Ser
             languageVersion,
             configurationName,
             extensions,
+            csharpLanguageVersion,
             UseConsolidatedMvcViews: useConsolidatedMvcViews,
-            SuppressAddComponentParameter: suppressAddComponentParameter);
+            SuppressAddComponentParameter: suppressAddComponentParameter,
+            UseRoslynTokenizer: useRoslynTokenizer,
+            PreprocessorSymbols: preprocessorSymbols);
     }
 
     public override void Serialize(ref MessagePackWriter writer, RazorConfiguration value, SerializerCachingOptions options)
     {
-        // Write 4 values + 1 value per extension.
+        // Write SerializerPropertyCount values + 1 value per extension.
         var extensions = value.Extensions;
-        var count = extensions.Length + 4;
+        var count = extensions.Length + SerializerPropertyCount;
 
         writer.WriteArrayHeader(count);
 
@@ -68,10 +78,13 @@ public override void Serialize(ref MessagePackWriter writer, RazorConfiguration
             CachedStringFormatter.Instance.Serialize(ref writer, value.LanguageVersion.ToString(), options);
         }
 
+        writer.Write((int)value.CSharpLanguageVersion);
         writer.Write(value.SuppressAddComponentParameter);
         writer.Write(value.UseConsolidatedMvcViews);
+        writer.Write(value.UseRoslynTokenizer);
+        writer.Serialize(value.PreprocessorSymbols, options);
 
-        count -= 4;
+        count -= SerializerPropertyCount;
 
         for (var i = 0; i < count; i++)
         {

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Serialization/MessagePack/SerializationFormat.cs

@@ -9,5 +9,5 @@ internal static class SerializationFormat
     // or any of the types that compose it changes. This includes: RazorConfiguration,
     // ProjectWorkspaceState, TagHelperDescriptor, and DocumentSnapshotHandle.
     // NOTE: If this version is changed, a coordinated insertion is required between Roslyn and Razor for the C# extension.
-    public const int Version = 6;
+    public const int Version = 7;
 }