Use apphost for core compiler (#80026)

* Use apphost for core compiler

* Remove unused members

* Include executables in the toolset package

* Avoid missing apphosts in source build

* Improve setting DOTNET_ROOT

* Skip invalid test combinations

* Fix linux and remove opt-out

* Fix tests

* Clarify

* Preserve executables through rehydration

* Revert unintentional change

* Fix DOTNET_ROOT setting

* Build macOS separately

* Add chmod on any unix

* Document s_executablesGroup

Author: jjonescz

azure-pipelines.yml

@@ -125,6 +125,10 @@ parameters:
     default:
       name: $(PoolName)
       demands: ImageOverride -equals $(WindowsQueueName)
+  - name: macOSPool
+    type: object
+    default:
+      vmImage: macOS-15
   - name: vs2022PreviewPool
     type: object
     default:
@@ -170,6 +174,16 @@ stages:
       testArtifactName: Transport_Artifacts_Unix_Debug
       poolParameters: ${{ parameters.ubuntuPool }}
 
+- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+  - stage: MacOS_Build
+    dependsOn: []
+    jobs:
+    - template: eng/pipelines/build-unix-job.yml
+      parameters:
+        jobName: Build_macOS_Debug
+        testArtifactName: Transport_Artifacts_macOS_Debug
+        poolParameters: ${{ parameters.macOSPool }}
+
 - stage: Source_Build
   dependsOn: []
   jobs:
@@ -370,14 +384,19 @@ stages:
         testArguments: --testCoreClr
         poolParameters: ${{ parameters.ubuntuPool }}
 
-  # https://github.com/dotnet/runtime/issues/97186
-  # Disabled until runtime can track down the crash
-  - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+  - stage: MacOS_Debug_CoreClr
+    dependsOn: MacOS_Build
+    variables:
+    - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      - group: DotNet-HelixApi-Access
+    jobs:
+    # https://github.com/dotnet/runtime/issues/97186
     - template: eng/pipelines/test-unix-job.yml
       parameters:
         testRunName: 'Test macOS Debug'
         jobName: Test_macOS_Debug
-        testArtifactName: Transport_Artifacts_Unix_Debug
+        testArtifactName: Transport_Artifacts_macOS_Debug
         configuration: Debug
         testArguments: --testCoreClr 
         helixQueueName: $(HelixMacOsQueueName)

eng/build.sh

@@ -341,6 +341,7 @@ function GetCompilerTestAssembliesIncludePaths {
   assemblies+=" --include '^Microsoft\.CodeAnalysis\.VisualBasic\.Emit\.UnitTests$'"
   assemblies+=" --include '^Roslyn\.Compilers\.VisualBasic\.IOperation\.UnitTests$'"
   assemblies+=" --include '^Microsoft\.CodeAnalysis\.VisualBasic\.CommandLine\.UnitTests$'"
+  assemblies+=" --include '^Microsoft\.Build\.Tasks\.CodeAnalysis\.UnitTests$'"
   echo "$assemblies"
 }
 

src/Compilers/CSharp/csc/AnyCpu/csc.csproj

@@ -4,7 +4,6 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFrameworks>$(NetRoslynSourceBuild);net472</TargetFrameworks>
-    <UseAppHost>false</UseAppHost>
     <IsSymbolPublishingPackage>true</IsSymbolPublishingPackage>
   </PropertyGroup>
 

src/Compilers/Core/MSBuildTask/InteractiveCompiler.cs

@@ -16,8 +16,6 @@ namespace Microsoft.CodeAnalysis.BuildTasks
     /// </summary>
     public abstract class InteractiveCompiler : ManagedToolTask
     {
-        internal readonly PropertyDictionary _store = new PropertyDictionary();
-
         public InteractiveCompiler()
             : base(ErrorString.ResourceManager)
         {

src/Compilers/Core/MSBuildTask/ManagedCompiler.cs

@@ -51,7 +51,6 @@ private enum CompilationKind
         }
 
         private CancellationTokenSource? _sharedCompileCts;
-        internal readonly PropertyDictionary _store = new PropertyDictionary();
 
         internal abstract RequestLanguage Language { get; }
 

src/Compilers/Core/MSBuildTask/ManagedToolTask.cs

@@ -4,18 +4,20 @@
 
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.IO;
 using System.Resources;
 using System.Text;
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
+using Microsoft.CodeAnalysis.CommandLine;
 using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis.BuildTasks
 {
     public abstract class ManagedToolTask : ToolTask
     {
+        internal readonly PropertyDictionary _store = new PropertyDictionary();
+
         /// <summary>
         /// A copy of this task, compiled for .NET Framework, is deployed into the .NET SDK. It is a bridge task
         /// that is loaded into .NET Framework MSBuild but launches the .NET Core compiler. This task necessarily
@@ -77,13 +79,7 @@ internal string GenerateToolArguments()
         /// </summary>
         protected sealed override string GenerateCommandLineCommands()
         {
-            var commandLineArguments = GenerateToolArguments();
-            if (UsingBuiltinTool && IsBuiltinToolRunningOnCoreClr)
-            {
-                commandLineArguments = RuntimeHostInfo.GetDotNetExecCommandLine(PathToBuiltInTool, commandLineArguments);
-            }
-
-            return commandLineArguments;
+            return GenerateToolArguments();
         }
 
         /// <summary>
@@ -118,12 +114,9 @@ protected sealed override string GenerateResponseFileCommands()
         /// This could be the managed assembly itself (on desktop .NET on Windows),
         /// or a runtime such as dotnet.
         /// </summary>
-        protected sealed override string GenerateFullPathToTool() => (UsingBuiltinTool, IsBuiltinToolRunningOnCoreClr) switch
-        {
-            (true, true) => RuntimeHostInfo.GetDotNetPathOrDefault(),
-            (true, false) => PathToBuiltInTool,
-            (false, _) => Path.Combine(ToolPath ?? "", ToolExe)
-        };
+        protected sealed override string GenerateFullPathToTool() => UsingBuiltinTool
+            ? PathToBuiltInTool
+            : Path.Combine(ToolPath ?? "", ToolExe);
 
         protected abstract string ToolNameWithoutExtension { get; }
 
@@ -139,10 +132,7 @@ protected sealed override string GenerateResponseFileCommands()
         /// It returns the name of the managed assembly, which might not be the path returned by
         /// GenerateFullPathToTool, which can return the path to e.g. the dotnet executable.
         /// </remarks>
-        protected sealed override string ToolName =>
-            IsBuiltinToolRunningOnCoreClr
-                ? $"{ToolNameWithoutExtension}.dll"
-                : $"{ToolNameWithoutExtension}.exe";
+        protected sealed override string ToolName => $"{ToolNameWithoutExtension}{PlatformInformation.ExeExtension}";
 
         /// <summary>
         /// This generates the command line arguments passed to the tool.
@@ -206,5 +196,17 @@ internal static string GetBuildTaskDirectory()
 
             return buildTaskDirectory;
         }
+
+        protected override bool ValidateParameters()
+        {
+            // Set DOTNET_ROOT so that the apphost executables launch properly.
+            if (RuntimeHostInfo.GetToolDotNetRoot() is { } dotNetRoot)
+            {
+                Log.LogMessage("Setting {0} to '{1}'", RuntimeHostInfo.DotNetRootEnvironmentName, dotNetRoot);
+                EnvironmentVariables = [.. EnvironmentVariables ?? [], $"{RuntimeHostInfo.DotNetRootEnvironmentName}={dotNetRoot}"];
+            }
+
+            return base.ValidateParameters();
+        }
     }
 }

src/Compilers/Core/MSBuildTaskTests/CscTests.cs

@@ -6,6 +6,7 @@
 using System.IO;
 using Microsoft.CodeAnalysis.BuildTasks.UnitTests.TestUtilities;
 using Roslyn.Test.Utilities;
+using Roslyn.Utilities;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -484,14 +485,13 @@ public void EmptyCscToolExe()
             csc.ToolExe = "";
             csc.Sources = MSBuildUtil.CreateTaskItems("test.cs");
             Assert.Equal("", csc.GenerateCommandLineContents());
-            // StartsWith because it can be csc.exe or csc.dll
-            Assert.StartsWith(Path.Combine("path", "to", "custom_csc", "csc."), csc.GeneratePathToTool());
+            AssertEx.Equal(Path.Combine("path", "to", "custom_csc", $"csc{PlatformInformation.ExeExtension}"), csc.GeneratePathToTool());
 
             csc = new Csc();
             csc.ToolPath = Path.Combine("path", "to", "custom_csc");
             csc.Sources = MSBuildUtil.CreateTaskItems("test.cs");
             Assert.Equal("", csc.GenerateCommandLineContents());
-            Assert.StartsWith(Path.Combine("path", "to", "custom_csc", "csc."), csc.GeneratePathToTool());
+            AssertEx.Equal(Path.Combine("path", "to", "custom_csc", $"csc{PlatformInformation.ExeExtension}"), csc.GeneratePathToTool());
         }
 
         [Fact]

src/Compilers/Core/MSBuildTaskTests/IntegrationTests.cs

@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-#if NET472
 using System.Collections.Generic;
 using System.IO;
 using Microsoft.CodeAnalysis.Test.Utilities;
@@ -24,24 +23,6 @@ private ProcessResult RunCompilerOutput(TempFile file)
             return ProcessUtilities.Run(file.Path, "", Path.GetDirectoryName(file.Path));
         }
 
-        private static void VerifyResult(ProcessResult result)
-        {
-            Assert.Equal("", result.Output);
-            Assert.Equal("", result.Errors);
-            Assert.Equal(0, result.ExitCode);
-        }
-
-        private void VerifyResultAndOutput(ProcessResult result, TempDirectory path, string expectedOutput)
-        {
-            using (var resultFile = GetResultFile(path, "hello.exe"))
-            {
-                VerifyResult(result);
-
-                var runningResult = RunCompilerOutput(resultFile);
-                Assert.Equal(expectedOutput, runningResult.Output);
-            }
-        }
-
         // A dictionary with name and contents of all the files we want to create for the SimpleMSBuild test.
         private Dictionary<string, string> SimpleMsBuildFiles => new Dictionary<string, string> {
 { "HelloSolution.sln",
@@ -536,7 +517,7 @@ public void ReportAnalyzerMSBuild()
             string arguments = string.Format(@"/m /nr:false /t:Rebuild /p:UseSharedCompilation=false /p:UseRoslyn=1 HelloSolution.sln");
             var result = RunCommandLineCompiler(_msbuildExecutable, arguments, _tempDirectory, ReportAnalyzerMsBuildFiles,
                 new Dictionary<string, string>
-                { { "MyMSBuildToolsPath", Path.GetDirectoryName(typeof(IntegrationTests).Assembly.Location) } });
+                { { "MyMSBuildToolsPath", Path.GetDirectoryName(typeof(IntegrationTests).Assembly.Location)! } });
 
             Assert.True(result.ExitCode != 0);
             Assert.Contains("/reportanalyzer", result.Output);
@@ -717,4 +698,3 @@ public class Class1
         }
     }
 }
-#endif

src/Compilers/Core/MSBuildTaskTests/MSBuildManagedToolTests.cs

@@ -2,9 +2,8 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using System.IO;
-using Microsoft.CodeAnalysis.Test.Utilities;
+using Roslyn.Utilities;
 using Xunit;
 
 namespace Microsoft.CodeAnalysis.BuildTasks.UnitTests;
@@ -16,7 +15,7 @@ public void PathToBuiltinTool()
     {
         var taskPath = Path.GetDirectoryName(typeof(ManagedCompiler).Assembly.Location)!;
         var relativePath = RuntimeHostInfo.IsCoreClrRuntime
-            ? Path.Combine("bincore", "csc.dll")
+            ? Path.Combine("bincore", $"csc{PlatformInformation.ExeExtension}")
             : "csc.exe";
         var task = new Csc();
         Assert.Equal(Path.Combine(taskPath, relativePath), task.PathToBuiltInTool);

src/Compilers/Core/MSBuildTaskTests/Microsoft.Build.Tasks.CodeAnalysis.UnitTests.csproj

@@ -69,6 +69,7 @@
   <Target Name="CopyAssetsForIntegrationTests" Condition="'@(CommandLineCompilerReference)' != ''" AfterTargets="ResolveProjectReferences" BeforeTargets="AssignTargetPaths">
     <PropertyGroup>
       <_CommandLineCompilerReferenceOutputPath>@(CommandLineCompilerReference->'%(RootDir)%(Directory)*.*')</_CommandLineCompilerReferenceOutputPath>
+      <_LinkPrefix Condition="'$(TargetFramework)' != 'net472'">bincore\</_LinkPrefix>
     </PropertyGroup>
     <ItemGroup>
       <_CommandLineCompilerReferenceContent Include="$(_CommandLineCompilerReferenceOutputPath)" />
@@ -78,7 +79,7 @@
       <Output TaskParameter="Filtered" ItemName="_CommandLineCompilerReferenceDeduplicated" />
     </RemoveDuplicates>
     <ItemGroup>
-      <Content Include="%(_CommandLineCompilerReferenceDeduplicated.Original)" CopyToOutputDirectory="PreserveNewest" />
+      <Content Include="%(_CommandLineCompilerReferenceDeduplicated.Original)" Link="$(_LinkPrefix)%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" />
     </ItemGroup>
   </Target>
 </Project>