feat: Add validator for benchmarks that contains null runtime

src/BenchmarkDotNet/Configs/DefaultConfig.cs

@@ -72,6 +72,7 @@ public IEnumerable<IValidator> GetValidators()
             yield return DeferredExecutionValidator.FailOnError;
             yield return ParamsAllValuesValidator.FailOnError;
             yield return ParamsValidator.FailOnError;
+            yield return RuntimeValidator.DontFailOnError;
         }
 
         public IOrderer Orderer => null;

src/BenchmarkDotNet/Validators/RuntimeValidator.cs

@@ -0,0 +1,43 @@
+using System.Collections.Generic;
+using System.Linq;
+using BenchmarkDotNet.Characteristics;
+
+namespace BenchmarkDotNet.Validators;
+
+/// <summary>
+/// Validator for runtime characteristic.
+/// <see href="https://github.com/dotnet/BenchmarkDotNet/issues/2609" />
+/// </summary>
+public class RuntimeValidator : IValidator
+{
+    public static readonly IValidator DontFailOnError = new RuntimeValidator();
+
+    private RuntimeValidator() { }
+
+    public bool TreatsWarningsAsErrors => false;
+
+    public IEnumerable<ValidationError> Validate(ValidationParameters input)
+    {
+        var allBenchmarks = input.Benchmarks.ToArray();
+        var nullRuntimeBenchmarks = allBenchmarks.Where(x => x.Job.Environment.Runtime == null).ToArray();
+
+        // There is no validation error if all the runtimes are set or if all the runtimes are null.
+        if (allBenchmarks.Length == nullRuntimeBenchmarks.Length)
+        {
+            return [];
+        }
+
+        var errors = new List<ValidationError>();
+        foreach (var benchmark in nullRuntimeBenchmarks)
+        {
+            var job = benchmark.Job;
+            var jobText = job.HasValue(CharacteristicObject.IdCharacteristic)
+                ? job.Id
+                : CharacteristicSetPresenter.Display.ToPresentation(job); // Use job text representation instead for auto generated JobId.
+
+            var message = $"Job({jobText}) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly.";
+            errors.Add(new ValidationError(false, message));
+        }
+        return errors;
+    }
+}

tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs

@@ -0,0 +1,125 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Validators;
+using System.Linq;
+using Xunit;
+
+namespace BenchmarkDotNet.Tests.Validators;
+
+public class RuntimeValidatorTests
+{
+    [Fact]
+    public void SameRuntime_Should_Success()
+    {
+        // Arrange
+        var config = new TestConfig1().CreateImmutableConfig();
+        var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config);
+        var parameters = new ValidationParameters(runInfo.BenchmarksCases, config);
+
+        // Act
+        var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray();
+
+        // Assert
+        Assert.Empty(errors);
+    }
+
+    [Fact]
+    public void NullRuntimeMixed_Should_Failed()
+    {
+        // Arrange
+        var config = new TestConfig2().CreateImmutableConfig();
+        var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config);
+        var parameters = new ValidationParameters(runInfo.BenchmarksCases, config);
+
+        // Act
+        var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray();
+
+        // Assert
+        {
+            var expectedMessage = "Job(Dry) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly.";
+            Assert.Contains(expectedMessage, errors);
+        }
+        {
+            var expectedMessage = "Job(Toolchain=.NET 10.0) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly.";
+            Assert.Contains(expectedMessage, errors);
+        }
+    }
+
+    [Fact]
+    public void NotNullRuntimeOnly_Should_Success()
+    {
+        // Arrange
+        var config = new TestConfig3().CreateImmutableConfig();
+        var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config);
+        var parameters = new ValidationParameters(runInfo.BenchmarksCases, config);
+
+        // Act
+        var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray();
+
+        // Assert
+        Assert.Empty(errors);
+    }
+
+    public class DummyBenchmark
+    {
+        [Benchmark]
+        public void Benchmark()
+        {
+        }
+    }
+
+    // TestConfig that expicitly specify runtime.
+    private class TestConfig1 : ManualConfig
+    {
+        public TestConfig1()
+        {
+            var baseJob = Job.Dry;
+
+            WithOption(ConfigOptions.DisableOptimizationsValidator, true);
+            AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray());
+
+            AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80));
+            AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90));
+        }
+    }
+
+    // TestConfig that contains job that don't specify runtime.
+    private class TestConfig2 : ManualConfig
+    {
+        public TestConfig2()
+        {
+            var baseJob = Job.Dry;
+
+            WithOption(ConfigOptions.DisableOptimizationsValidator, true);
+            AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray());
+
+            AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80));
+            AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90)
+                          .WithRuntime(CoreRuntime.Core90));
+
+            // Validate error message for auto generated jobid.
+            AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0));
+        }
+    }
+
+    // TestConfig that expicitly specify runtime.
+    private class TestConfig3 : ManualConfig
+    {
+        public TestConfig3()
+        {
+            var baseJob = Job.Dry;
+
+            WithOption(ConfigOptions.DisableOptimizationsValidator, true);
+            AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray());
+
+            AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)
+                          .WithRuntime(CoreRuntime.Core80)); ;
+            AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90)
+                          .WithRuntime(CoreRuntime.Core90));
+        }
+    }
+}