From 2015aa43a57e59a90b59407798090db59270b7fe Mon Sep 17 00:00:00 2001 From: jnm2 Date: Sat, 31 May 2025 15:44:23 -0400 Subject: [PATCH 1/6] Enable running example tests massively in parallel via test runner UI and dotnet test --- tools/ExampleExtractor/ExampleMetadata.cs | 2 ++ tools/ExampleTester/ExampleTester.csproj | 5 +++- tools/ExampleTester/ExampleTests.cs | 35 ++++++++++++++++++++++ tools/ExampleTester/GeneratedExample.cs | 2 ++ tools/ExampleTester/TesterConfiguration.cs | 9 ++---- 5 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 tools/ExampleTester/ExampleTests.cs diff --git a/tools/ExampleExtractor/ExampleMetadata.cs b/tools/ExampleExtractor/ExampleMetadata.cs index a5b79fc56..cd52a7cbe 100644 --- a/tools/ExampleExtractor/ExampleMetadata.cs +++ b/tools/ExampleExtractor/ExampleMetadata.cs @@ -84,4 +84,6 @@ public class ExampleMetadata [JsonIgnore] public string Source => $"{MarkdownFile}:{StartLine}-{EndLine}"; + + public override string ToString() => Name; } diff --git a/tools/ExampleTester/ExampleTester.csproj b/tools/ExampleTester/ExampleTester.csproj index 46f73c0cd..9d358e78e 100644 --- a/tools/ExampleTester/ExampleTester.csproj +++ b/tools/ExampleTester/ExampleTester.csproj @@ -1,4 +1,4 @@ - + Exe @@ -11,6 +11,9 @@ + + + diff --git a/tools/ExampleTester/ExampleTests.cs b/tools/ExampleTester/ExampleTests.cs new file mode 100644 index 000000000..5b59790ff --- /dev/null +++ b/tools/ExampleTester/ExampleTests.cs @@ -0,0 +1,35 @@ +using NUnit.Framework; +using Utilities; + +[assembly: Parallelizable(ParallelScope.Children)] + +namespace ExampleTester; + +public static class ExampleTests +{ + private static TesterConfiguration TesterConfiguration { get; } = new(FindTmpDirectory()); + private static StatusCheckLogger StatusCheckLogger { get; } = new("..", "Example tester"); + + public static IEnumerable LoadExamples() => + from example in GeneratedExample.LoadAllExamples(TesterConfiguration.ExtractedOutputDirectory) + select new object[] { example }; + + [TestCaseSource(nameof(LoadExamples))] + public static async Task ExamplePasses(GeneratedExample example) + { + if (!await example.Test(TesterConfiguration, StatusCheckLogger)) + Assert.Fail("There were one or more failures. See the logged output for details."); + } + + private static string FindTmpDirectory() + { + for (string? current = AppDomain.CurrentDomain.BaseDirectory; current != null; current = Path.GetDirectoryName(current)) + { + string testPath = Path.Join(current, "tmp"); + if (Directory.Exists(testPath)) + return testPath; + } + + throw new InvalidOperationException($"Can't find 'tmp' directory in {AppDomain.CurrentDomain.BaseDirectory} or any parent directories."); + } +} diff --git a/tools/ExampleTester/GeneratedExample.cs b/tools/ExampleTester/GeneratedExample.cs index 6c06eab33..c7928df1e 100644 --- a/tools/ExampleTester/GeneratedExample.cs +++ b/tools/ExampleTester/GeneratedExample.cs @@ -26,6 +26,8 @@ private GeneratedExample(string directory) Metadata = JsonConvert.DeserializeObject(metadataJson) ?? throw new ArgumentException($"Invalid (null) metadata in {directory}"); } + public override string? ToString() => Metadata.ToString(); + internal static List LoadAllExamples(string parentDirectory) => Directory.GetDirectories(parentDirectory).Select(Load).ToList(); diff --git a/tools/ExampleTester/TesterConfiguration.cs b/tools/ExampleTester/TesterConfiguration.cs index b6a0c43d9..eee1dff14 100644 --- a/tools/ExampleTester/TesterConfiguration.cs +++ b/tools/ExampleTester/TesterConfiguration.cs @@ -5,12 +5,9 @@ namespace ExampleTester; public record TesterConfiguration( string ExtractedOutputDirectory, - bool Quiet, - string? SourceFile, - string? ExampleName) -{ - -} + bool Quiet = false, + string? SourceFile = null, + string? ExampleName = null); public class TesterConfigurationBinder : BinderBase { From beb677f57ba51b36a2542be9824f0fd5d367bf80 Mon Sep 17 00:00:00 2001 From: jnm2 Date: Sat, 31 May 2025 16:07:48 -0400 Subject: [PATCH 2/6] Fix concurrency issues over Console.Out --- tools/ExampleTester/ExampleTests.cs | 5 +- tools/ExampleTester/GeneratedExample.cs | 68 +++++++++++++----------- tools/ExampleTester/Program.cs | 2 +- tools/MarkdownConverter/Spec/Reporter.cs | 2 +- tools/StandardAnchorTags/Program.cs | 2 +- tools/Utilities/StatusCheckLogger.cs | 10 ++-- 6 files changed, 49 insertions(+), 40 deletions(-) diff --git a/tools/ExampleTester/ExampleTests.cs b/tools/ExampleTester/ExampleTests.cs index 5b59790ff..0c1d29b10 100644 --- a/tools/ExampleTester/ExampleTests.cs +++ b/tools/ExampleTester/ExampleTests.cs @@ -8,7 +8,6 @@ namespace ExampleTester; public static class ExampleTests { private static TesterConfiguration TesterConfiguration { get; } = new(FindTmpDirectory()); - private static StatusCheckLogger StatusCheckLogger { get; } = new("..", "Example tester"); public static IEnumerable LoadExamples() => from example in GeneratedExample.LoadAllExamples(TesterConfiguration.ExtractedOutputDirectory) @@ -17,7 +16,9 @@ from example in GeneratedExample.LoadAllExamples(TesterConfiguration.ExtractedOu [TestCaseSource(nameof(LoadExamples))] public static async Task ExamplePasses(GeneratedExample example) { - if (!await example.Test(TesterConfiguration, StatusCheckLogger)) + var logger = new StatusCheckLogger(TestContext.Out, "..", "Example tester"); + + if (!await example.Test(TesterConfiguration, logger)) Assert.Fail("There were one or more failures. See the logged output for details."); } diff --git a/tools/ExampleTester/GeneratedExample.cs b/tools/ExampleTester/GeneratedExample.cs index c7928df1e..4c1a4a851 100644 --- a/tools/ExampleTester/GeneratedExample.cs +++ b/tools/ExampleTester/GeneratedExample.cs @@ -9,8 +9,10 @@ namespace ExampleTester; -internal class GeneratedExample +public class GeneratedExample { + private static readonly object ConsoleAccessLock = new(); + static GeneratedExample() { MSBuildLocator.RegisterDefaults(); @@ -135,46 +137,52 @@ bool ValidateOutput() ? new object[] { Metadata.ExecutionArgs ?? new string[0] } : new object[0]; - var oldOut = Console.Out; List actualLines; Exception? actualException = null; - try + lock (ConsoleAccessLock) { - var builder = new StringBuilder(); - Console.SetOut(new StringWriter(builder)); + var oldOut = Console.Out; try { - var result = method.Invoke(null, arguments); - // For async Main methods, the compilation's entry point is still the Main - // method, so we explicitly wait for the returned task just like the synthesized - // entry point would. - if (result is Task task) + var builder = new StringBuilder(); + Console.SetOut(new StringWriter(builder)); + try + { + var result = method.Invoke(null, arguments); + // For async Main methods, the compilation's entry point is still the Main + // method, so we explicitly wait for the returned task just like the synthesized + // entry point would. + if (result is Task task) + { + task.GetAwaiter().GetResult(); + } + + // For some reason, we don't *actually* get the result of all finalizers + // without this. We shouldn't need it (as relevant examples already have it) but + // code that works outside the test harness doesn't work inside it. + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + catch (TargetInvocationException outer) { - task.GetAwaiter().GetResult(); + actualException = outer.InnerException ?? throw new InvalidOperationException("TargetInvocationException had no nested exception"); } - // For some reason, we don't *actually* get the result of all finalizers - // without this. We shouldn't need it (as relevant examples already have it) but - // code that works outside the test harness doesn't work inside it. - GC.Collect(); - GC.WaitForPendingFinalizers(); + + // Skip blank lines, to avoid unnecessary trailing empties. + // Also trim the end of each actual line, to avoid trailing spaces being necessary in the metadata + // or listed console output. + actualLines = builder.ToString() + .Replace("\r\n", "\n") + .Split('\n') + .Select(line => line.TrimEnd()) + .Where(line => line != "").ToList(); } - catch (TargetInvocationException outer) + finally { - actualException = outer.InnerException ?? throw new InvalidOperationException("TargetInvocationException had no nested exception"); + Console.SetOut(oldOut); } - // Skip blank lines, to avoid unnecessary trailing empties. - // Also trim the end of each actual line, to avoid trailing spaces being necessary in the metadata - // or listed console output. - actualLines = builder.ToString() - .Replace("\r\n", "\n") - .Split('\n') - .Select(line => line.TrimEnd()) - .Where(line => line != "").ToList(); - } - finally - { - Console.SetOut(oldOut); } + var expectedLines = Metadata.ExpectedOutput ?? new List(); return ValidateException(actualException, Metadata.ExpectedException) && (Metadata.IgnoreOutput || ValidateExpectedAgainstActual("output", expectedLines, actualLines)); diff --git a/tools/ExampleTester/Program.cs b/tools/ExampleTester/Program.cs index 937a79e3d..7486d036a 100644 --- a/tools/ExampleTester/Program.cs +++ b/tools/ExampleTester/Program.cs @@ -2,7 +2,7 @@ using System.CommandLine; using Utilities; -var logger = new StatusCheckLogger("..", "Example tester"); +var logger = new StatusCheckLogger(Console.Out, "..", "Example tester"); var headSha = Environment.GetEnvironmentVariable("HEAD_SHA"); var token = Environment.GetEnvironmentVariable("GH_TOKEN"); diff --git a/tools/MarkdownConverter/Spec/Reporter.cs b/tools/MarkdownConverter/Spec/Reporter.cs index 4adaa52c8..58930aa45 100644 --- a/tools/MarkdownConverter/Spec/Reporter.cs +++ b/tools/MarkdownConverter/Spec/Reporter.cs @@ -28,7 +28,7 @@ public Reporter() : this(null, null) { } public Reporter(Reporter? parent, string? filename) { // This is needed so that all Reporters share the same GitHub logger. - this.githubLogger = parent?.githubLogger ?? new StatusCheckLogger("..", "Markdown to Word Converter"); + this.githubLogger = parent?.githubLogger ?? new StatusCheckLogger(Console.Out, "..", "Markdown to Word Converter"); this.parent = parent; Location = new SourceLocation(filename, null, null, null); } diff --git a/tools/StandardAnchorTags/Program.cs b/tools/StandardAnchorTags/Program.cs index 6c0f77449..fea1ffd7b 100644 --- a/tools/StandardAnchorTags/Program.cs +++ b/tools/StandardAnchorTags/Program.cs @@ -25,7 +25,7 @@ public class Program /// 0 on success, non-zero on failure static async Task Main(string owner, string repo, bool dryrun =false) { - var logger = new StatusCheckLogger("..", "TOC and Anchor updater"); + var logger = new StatusCheckLogger(Console.Out, "..", "TOC and Anchor updater"); var headSha = Environment.GetEnvironmentVariable("HEAD_SHA"); var token = Environment.GetEnvironmentVariable("GH_TOKEN"); using FileStream openStream = File.OpenRead(FilesPath); diff --git a/tools/Utilities/StatusCheckLogger.cs b/tools/Utilities/StatusCheckLogger.cs index 2feaeb65c..dc8e83c4f 100644 --- a/tools/Utilities/StatusCheckLogger.cs +++ b/tools/Utilities/StatusCheckLogger.cs @@ -22,7 +22,7 @@ public record StatusCheckMessage(string file, int StartLine, int EndLine, string /// /// The path to the root of the repository /// The name of the tool that is running the check -public class StatusCheckLogger(string pathToRoot, string toolName) +public class StatusCheckLogger(TextWriter writer, string pathToRoot, string toolName) { private List annotations = []; public bool Success { get; private set; } = true; @@ -30,7 +30,7 @@ public class StatusCheckLogger(string pathToRoot, string toolName) // Utility method to format the path to unix style, from the root of the repository. private string FormatPath(string path) => Path.GetRelativePath(pathToRoot, path).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); - private void WriteMessageToConsole(string prefix, StatusCheckMessage d) => Console.WriteLine($"{prefix}{toolName}-{d.Id}::file={FormatPath(d.file)},line={d.StartLine}::{d.Message}"); + private void WriteMessageToConsole(string prefix, StatusCheckMessage d) => writer.WriteLine($"{prefix}{toolName}-{d.Id}::file={FormatPath(d.file)},line={d.StartLine}::{d.Message}"); /// /// Log a notice from the status check to the console only @@ -178,9 +178,9 @@ public async Task BuildCheckRunResult(string token, string owner, string repo, s // Once running on a branch on the dotnet org, this should work correctly. catch (ForbiddenException e) { - Console.WriteLine("===== WARNING: Could not create a check run.====="); - Console.WriteLine("Exception details:"); - Console.WriteLine(e); + writer.WriteLine("===== WARNING: Could not create a check run.====="); + writer.WriteLine("Exception details:"); + writer.WriteLine(e); } } } From 509dfab28788384325c76412599154480d3cf036 Mon Sep 17 00:00:00 2001 From: jnm2 Date: Sun, 1 Jun 2025 09:17:22 -0400 Subject: [PATCH 3/6] Fix warning CS7022 "The entry point of the program is global code; ignoring 'AutoGeneratedProgram.Main(string[])' entry point." --- tools/ExampleTester/ExampleTester.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ExampleTester/ExampleTester.csproj b/tools/ExampleTester/ExampleTester.csproj index 9d358e78e..ce9571731 100644 --- a/tools/ExampleTester/ExampleTester.csproj +++ b/tools/ExampleTester/ExampleTester.csproj @@ -5,6 +5,7 @@ net8.0 enable enable + false From 879fa129ebd44278b360106058f072cc4b22f419 Mon Sep 17 00:00:00 2001 From: jnm2 Date: Sun, 1 Jun 2025 09:43:49 -0400 Subject: [PATCH 4/6] More understandable failure when tools/tmp doesn't exist --- tools/ExampleTester/ExampleTests.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/ExampleTester/ExampleTests.cs b/tools/ExampleTester/ExampleTests.cs index 0c1d29b10..f772bc032 100644 --- a/tools/ExampleTester/ExampleTests.cs +++ b/tools/ExampleTester/ExampleTests.cs @@ -7,7 +7,7 @@ namespace ExampleTester; public static class ExampleTests { - private static TesterConfiguration TesterConfiguration { get; } = new(FindTmpDirectory()); + private static TesterConfiguration TesterConfiguration { get; } = new(Path.Join(FindSlnDirectory(), "tmp")); public static IEnumerable LoadExamples() => from example in GeneratedExample.LoadAllExamples(TesterConfiguration.ExtractedOutputDirectory) @@ -22,15 +22,14 @@ public static async Task ExamplePasses(GeneratedExample example) Assert.Fail("There were one or more failures. See the logged output for details."); } - private static string FindTmpDirectory() + private static string FindSlnDirectory() { for (string? current = AppDomain.CurrentDomain.BaseDirectory; current != null; current = Path.GetDirectoryName(current)) { - string testPath = Path.Join(current, "tmp"); - if (Directory.Exists(testPath)) - return testPath; + if (Directory.EnumerateFiles(current, "*.sln").Any()) + return current; } - throw new InvalidOperationException($"Can't find 'tmp' directory in {AppDomain.CurrentDomain.BaseDirectory} or any parent directories."); + throw new InvalidOperationException($"Can't find a directory containing a .sln file in {AppDomain.CurrentDomain.BaseDirectory} or any parent directories."); } } From e94690b5bc3c0a300900fddbb3e01750b1dde8b1 Mon Sep 17 00:00:00 2001 From: jnm2 Date: Sun, 1 Jun 2025 09:47:06 -0400 Subject: [PATCH 5/6] Don't run ExampleTester unit tests since it's covered in a different run --- .github/workflows/tools-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tools-tests.yaml b/.github/workflows/tools-tests.yaml index 9695e88b4..149613221 100644 --- a/.github/workflows/tools-tests.yaml +++ b/.github/workflows/tools-tests.yaml @@ -33,4 +33,4 @@ jobs: - name: Run all tests run: | cd tools - dotnet test + dotnet test --filter Name!~ExampleTests From e46e108f16dcb4f33f5e3f17448add5d3901e043 Mon Sep 17 00:00:00 2001 From: jnm2 Date: Tue, 9 Sep 2025 18:51:44 -0400 Subject: [PATCH 6/6] Clarify lock purpose in variable name --- tools/ExampleTester/GeneratedExample.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ExampleTester/GeneratedExample.cs b/tools/ExampleTester/GeneratedExample.cs index 68a3b19cc..6df3d4955 100644 --- a/tools/ExampleTester/GeneratedExample.cs +++ b/tools/ExampleTester/GeneratedExample.cs @@ -10,7 +10,7 @@ namespace ExampleTester; public class GeneratedExample { - private static readonly object ConsoleAccessLock = new(); + private static readonly object CodeExecutionLock = new(); private readonly string directory; internal ExampleMetadata Metadata { get; } @@ -133,7 +133,7 @@ bool ValidateOutput() List actualLines; Exception? actualException = null; - lock (ConsoleAccessLock) + lock (CodeExecutionLock) { var oldOut = Console.Out; try