From bf2380ee53c5e70a744c5701bd7c247853e32eaa Mon Sep 17 00:00:00 2001 From: Bert Date: Tue, 10 Oct 2023 10:02:20 +0200 Subject: [PATCH 1/5] fix deadlog in BaseTest --- Directory.Build.props | 5 +++ Directory.Packages.props | 1 + eng/publish-coverlet-result-files.yml | 42 ++++++------------- .../coverlet.core.tests.csproj | 29 +++++++------ ...verlet.integration.determisticbuild.csproj | 1 + test/coverlet.integration.tests/BaseTest.cs | 34 ++++++++++----- .../DeterministicBuild.cs | 6 ++- 7 files changed, 64 insertions(+), 54 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 9373b11bf..8d701ede9 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -25,4 +25,9 @@ true + + + + + diff --git a/Directory.Packages.props b/Directory.Packages.props index eb24d12bb..55b53e2b4 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -10,6 +10,7 @@ + diff --git a/eng/publish-coverlet-result-files.yml b/eng/publish-coverlet-result-files.yml index 81aa587bd..536ac2506 100644 --- a/eng/publish-coverlet-result-files.yml +++ b/eng/publish-coverlet-result-files.yml @@ -1,36 +1,20 @@ steps: -- task: Powershell@2 - displayName: Prepare log files to Upload - inputs: - targetType: inline - pwsh: true - script: | - New-Item -ItemType Directory "$(Build.SourcesDirectory)/artifacts/TestLogs/" - function Copy-FileKeepPath { - param ( - $Filter,$FileToCopy,$Destination,$StartIndex - ) - Get-ChildItem -Path $FileToCopy -Filter $Filter -Recurse -File | ForEach-Object { - $fileName = $_.FullName - $newDestination=Join-Path -Path $Destination -ChildPath $fileName.Substring($StartIndex) - $folder=Split-Path -Path $newDestination -Parent - if (!(Test-Path -Path $folder)) {New-Item $folder -Type Directory -Force -Verbose} - Copy-Item -Path $fileName -Destination $newDestination -Recurse -Force -Verbose - } - } - $logfiles = "coverage.opencover.xml", "coverage.cobertura.xml", "coverage.json", "log.txt", "log.datacollector.*.txt", "log.host.*.txt" - foreach ($logfile in $logfiles ) { - Copy-FileKeepPath -FileToCopy "$(Build.SourcesDirectory)/test/*" -des "$(Build.SourcesDirectory)/artifacts/TestLogs/" -filter $logfile -startIndex "$(Build.SourcesDirectory)/test/coverlet.integration.tests/bin/".Length - } - +- task: CopyFiles@2 + displayName: Copy tests results continueOnError: true condition: always() - -- task: CopyFiles@2 - displayName: Copy trx files inputs: - SourceFolder: '$(Agent.TempDirectory)' - Contents: '**/*.trx' + SourceFolder: '$(Build.SourcesDirectory)/artifacts' + Contents: | + **/*.trx + **/*.html + **/*.binlog + **/coverage.opencover.xml + **/coverage.cobertura.xml + **/coverage.json + **/log.txt + **/log.datacollector.*.txt + **/log.host.*.txt TargetFolder: '$(Build.SourcesDirectory)/artifacts/TestLogs' - task: PublishPipelineArtifact@1 diff --git a/test/coverlet.core.tests/coverlet.core.tests.csproj b/test/coverlet.core.tests/coverlet.core.tests.csproj index bfb8acd87..59a26a1aa 100644 --- a/test/coverlet.core.tests/coverlet.core.tests.csproj +++ b/test/coverlet.core.tests/coverlet.core.tests.csproj @@ -8,29 +8,34 @@ NU1702 true + false - - - - - - - - - - + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + - + - + diff --git a/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj b/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj index 8aca96c96..88481cd5a 100644 --- a/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj +++ b/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj @@ -24,5 +24,6 @@ + \ No newline at end of file diff --git a/test/coverlet.integration.tests/BaseTest.cs b/test/coverlet.integration.tests/BaseTest.cs index 2d44555e0..01013d845 100644 --- a/test/coverlet.integration.tests/BaseTest.cs +++ b/test/coverlet.integration.tests/BaseTest.cs @@ -96,19 +96,27 @@ private protected ClonedTemplateProject CloneTemplateProject(bool cleanupOnDispo private protected bool RunCommand(string command, string arguments, out string standardOutput, out string standardError, string workingDirectory = "") { Debug.WriteLine($"BaseTest.RunCommand: {command} {arguments}\nWorkingDirectory: {workingDirectory}"); - var psi = new ProcessStartInfo(command, arguments); - psi.WorkingDirectory = workingDirectory; - psi.RedirectStandardError = true; - psi.RedirectStandardOutput = true; - Process commandProcess = Process.Start(psi)!; + // https://learn.microsoft.com/en-us/dotnet/api/system.diagnostics.process.standardoutput?view=net-7.0&redirectedfrom=MSDN#System_Diagnostics_Process_StandardOutput + var commandProcess = new Process(); + commandProcess.StartInfo.FileName = command; + commandProcess.StartInfo.Arguments = arguments; + commandProcess.StartInfo.WorkingDirectory = workingDirectory; + commandProcess.StartInfo.RedirectStandardError = true; + commandProcess.StartInfo.RedirectStandardOutput = true; + commandProcess.StartInfo.UseShellExecute = false; + string eOut = ""; + commandProcess.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => { eOut += e.Data; }); + commandProcess.Start(); + // To avoid deadlocks, use an asynchronous read operation on at least one of the streams. + commandProcess.BeginErrorReadLine(); + standardOutput = commandProcess.StandardOutput.ReadToEnd(); if (!commandProcess.WaitForExit((int)TimeSpan.FromMinutes(5).TotalMilliseconds)) { - throw new XunitException($"Command 'dotnet {arguments}' didn't end after 5 minute"); + throw new XunitException($"Command 'dotnet {arguments}' didn't end after 5 minute"); } - standardOutput = commandProcess.StandardOutput.ReadToEnd(); - standardError = commandProcess.StandardError.ReadToEnd(); + standardError = eOut; return commandProcess.ExitCode == 0; - } + } private protected bool DotnetCli(string arguments, out string standardOutput, out string standardError, string workingDirectory = "") { @@ -192,7 +200,9 @@ private protected void AddCoverletMsbuildRef(string projectPath) string msbuildPkgVersion = GetPackageVersion("*msbuild*.nupkg"); xml.Element("Project")! .Element("ItemGroup")! - .Add(new XElement("PackageReference", new XAttribute("Include", "coverlet.msbuild"), new XAttribute("Version", msbuildPkgVersion))); + .Add(new XElement("PackageReference", new XAttribute("Include", "coverlet.msbuild"), new XAttribute("Version", msbuildPkgVersion), + new XElement("PrivateAssets", "all"), + new XElement("IncludeAssets", "runtime; build; native; contentfiles; analyzers"))); xml.Save(csproj); } @@ -211,7 +221,9 @@ private protected void AddCoverletCollectosRef(string projectPath) string msbuildPkgVersion = GetPackageVersion("*collector*.nupkg"); xml.Element("Project")! .Element("ItemGroup")! - .Add(new XElement("PackageReference", new XAttribute("Include", "coverlet.collector"), new XAttribute("Version", msbuildPkgVersion))); + .Add(new XElement("PackageReference", new XAttribute("Include", "coverlet.collector"), new XAttribute("Version", msbuildPkgVersion), + new XElement("PrivateAssets", "all"), + new XElement("IncludeAssets", "runtime; build; native; contentfiles; analyzers"))); xml.Save(csproj); } diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs index d5f02f39d..5559a62ec 100644 --- a/test/coverlet.integration.tests/DeterministicBuild.cs +++ b/test/coverlet.integration.tests/DeterministicBuild.cs @@ -149,7 +149,7 @@ public void Collectors() Assert.Contains("=/_/", File.ReadAllText(sourceRootMappingFilePath)); string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought", deterministicReport: true); - Assert.True(DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError), standardOutput); + bool result = DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError); if (!string.IsNullOrEmpty(standardError)) { _output.WriteLine(standardError); @@ -158,6 +158,7 @@ public void Collectors() { _output.WriteLine(standardOutput); } + Assert.True(result); Assert.Contains("Passed!", standardOutput); AssertCoverage(standardOutput); @@ -185,7 +186,7 @@ public void Collectors_SourceLink() Assert.Contains("=/_/", File.ReadAllText(sourceRootMappingFilePath)); string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought", sourceLink: true); - Assert.True(DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError), standardOutput); + bool result = DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError); if (!string.IsNullOrEmpty(standardError)) { _output.WriteLine(standardError); @@ -194,6 +195,7 @@ public void Collectors_SourceLink() { _output.WriteLine(standardOutput); } + Assert.True(result); Assert.Contains("Passed!", standardOutput); AssertCoverage(standardOutput, checkDeterministicReport: false); Assert.Contains("raw.githubusercontent.com", File.ReadAllText(Directory.GetFiles(_testProjectPath, "coverage.cobertura.xml", SearchOption.AllDirectories).Single())); From ccd7d0e9d1c547e308e1f0b47e0df0720e95d8cc Mon Sep 17 00:00:00 2001 From: Bert Date: Tue, 10 Oct 2023 10:27:55 +0200 Subject: [PATCH 2/5] remove PackageReference --- .../coverlet.integration.determisticbuild.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj b/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj index 88481cd5a..8aca96c96 100644 --- a/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj +++ b/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj @@ -24,6 +24,5 @@ - \ No newline at end of file From 23b8687d5a2ae2f3f23e637488d5cce48bdf3640 Mon Sep 17 00:00:00 2001 From: Bert Date: Tue, 10 Oct 2023 10:33:43 +0200 Subject: [PATCH 3/5] fix path --- eng/publish-coverlet-result-files.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/publish-coverlet-result-files.yml b/eng/publish-coverlet-result-files.yml index 536ac2506..51034eb2f 100644 --- a/eng/publish-coverlet-result-files.yml +++ b/eng/publish-coverlet-result-files.yml @@ -4,7 +4,7 @@ steps: continueOnError: true condition: always() inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' + SourceFolder: '$(Build.SourcesDirectory)/test' Contents: | **/*.trx **/*.html From b306c4a2b0032dca32cd82c9bf123f89c028e9f0 Mon Sep 17 00:00:00 2001 From: Bert Date: Tue, 10 Oct 2023 10:46:03 +0200 Subject: [PATCH 4/5] copy from Agent.TempDirectory --- eng/publish-coverlet-result-files.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/eng/publish-coverlet-result-files.yml b/eng/publish-coverlet-result-files.yml index 51034eb2f..102ede1dd 100644 --- a/eng/publish-coverlet-result-files.yml +++ b/eng/publish-coverlet-result-files.yml @@ -6,7 +6,6 @@ steps: inputs: SourceFolder: '$(Build.SourcesDirectory)/test' Contents: | - **/*.trx **/*.html **/*.binlog **/coverage.opencover.xml @@ -17,6 +16,13 @@ steps: **/log.host.*.txt TargetFolder: '$(Build.SourcesDirectory)/artifacts/TestLogs' +- task: CopyFiles@2 + displayName: Copy trx files + inputs: + SourceFolder: '$(Agent.TempDirectory)' + Contents: '**/*.trx' + TargetFolder: '$(Build.SourcesDirectory)/artifacts/TestLogs' + - task: PublishPipelineArtifact@1 displayName: Publish Coverlet logs continueOnError: true From db90d420338d0d98ff552d54d4b340ff3a947a6a Mon Sep 17 00:00:00 2001 From: Bert Date: Tue, 10 Oct 2023 12:38:53 +0200 Subject: [PATCH 5/5] new location for test results "VSTestResultsDirectory" --- Directory.Build.props | 5 ++ eng/publish-coverlet-result-files.yml | 3 +- .../DeterministicBuild.cs | 88 +++++++++++-------- 3 files changed, 58 insertions(+), 38 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 8d701ede9..d44607ea9 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -30,4 +30,9 @@ + + $(RepoRoot)/artifacts/tests + @(VSTestLogger) + + diff --git a/eng/publish-coverlet-result-files.yml b/eng/publish-coverlet-result-files.yml index 102ede1dd..06094d13e 100644 --- a/eng/publish-coverlet-result-files.yml +++ b/eng/publish-coverlet-result-files.yml @@ -4,8 +4,9 @@ steps: continueOnError: true condition: always() inputs: - SourceFolder: '$(Build.SourcesDirectory)/test' + SourceFolder: '$(Build.SourcesDirectory)' Contents: | + **/*.trx **/*.html **/*.binlog **/coverage.opencover.xml diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs index 5559a62ec..2a85dbadc 100644 --- a/test/coverlet.integration.tests/DeterministicBuild.cs +++ b/test/coverlet.integration.tests/DeterministicBuild.cs @@ -46,7 +46,7 @@ private protected void AssertCoverage(string standardOutput = "", bool checkDete { bool coverageChecked = false; string reportFilePath = ""; - foreach (string coverageFile in Directory.GetFiles(_testProjectPath, "coverage.json", SearchOption.AllDirectories)) + foreach (string coverageFile in Directory.GetFiles(GetReportPath(standardOutput), "coverage.json", SearchOption.AllDirectories)) { Classes? document = JsonConvert.DeserializeObject(File.ReadAllText(coverageFile))?.Document("DeepThought.cs"); if (document != null) @@ -65,7 +65,7 @@ private protected void AssertCoverage(string standardOutput = "", bool checkDete if (checkDeterministicReport) { // Verify deterministic report - foreach (string coverageFile in Directory.GetFiles(_testProjectPath, "coverage.cobertura.xml", SearchOption.AllDirectories)) + foreach (string coverageFile in Directory.GetFiles(GetReportPath(standardOutput), "coverage.cobertura.xml", SearchOption.AllDirectories)) { Assert.Contains("/_/test/coverlet.integration.determisticbuild/DeepThought.cs", File.ReadAllText(coverageFile)); File.Delete(coverageFile); @@ -176,43 +176,57 @@ public void Collectors() [Fact] public void Collectors_SourceLink() - { - CreateDeterministicTestPropsFile(); - DotnetCli($"build -c {_buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); - Assert.Contains("Build succeeded.", standardOutput); - string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping_coverletsample.integration.determisticbuild"); - Assert.True(File.Exists(sourceRootMappingFilePath), sourceRootMappingFilePath); - Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath)); - Assert.Contains("=/_/", File.ReadAllText(sourceRootMappingFilePath)); - - string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought", sourceLink: true); - bool result = DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError); - if (!string.IsNullOrEmpty(standardError)) - { - _output.WriteLine(standardError); - } - else - { - _output.WriteLine(standardOutput); - } - Assert.True(result); - Assert.Contains("Passed!", standardOutput); - AssertCoverage(standardOutput, checkDeterministicReport: false); - Assert.Contains("raw.githubusercontent.com", File.ReadAllText(Directory.GetFiles(_testProjectPath, "coverage.cobertura.xml", SearchOption.AllDirectories).Single())); - - // Check out/in process collectors injection - string dataCollectorLogContent = File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.datacollector.*.txt").Single()); - Assert.Contains("[coverlet]Initializing CoverletCoverageDataCollector with configuration:", dataCollectorLogContent); - Assert.Contains("[coverlet]Initialize CoverletInProcDataCollector", File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.host.*.txt").Single())); - Assert.Contains("[coverlet]Mapping resolved", dataCollectorLogContent); + { + CreateDeterministicTestPropsFile(); + DotnetCli($"build -c {_buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); + Assert.Contains("Build succeeded.", standardOutput); + string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping_coverletsample.integration.determisticbuild"); + Assert.True(File.Exists(sourceRootMappingFilePath), sourceRootMappingFilePath); + Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath)); + Assert.Contains("=/_/", File.ReadAllText(sourceRootMappingFilePath)); + + string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought", sourceLink: true); + bool result = DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError); + if (!string.IsNullOrEmpty(standardError)) + { + _output.WriteLine(standardError); + } + else + { + _output.WriteLine(standardOutput); + } + Assert.True(result); + Assert.Contains("Passed!", standardOutput); + AssertCoverage(standardOutput, checkDeterministicReport: false); + Assert.Contains("raw.githubusercontent.com", File.ReadAllText(Directory.GetFiles(GetReportPath(standardOutput), "coverage.cobertura.xml", SearchOption.AllDirectories).Single())); + + // Check out/in process collectors injection + string dataCollectorLogContent = File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.datacollector.*.txt").Single()); + Assert.Contains("[coverlet]Initializing CoverletCoverageDataCollector with configuration:", dataCollectorLogContent); + Assert.Contains("[coverlet]Initialize CoverletInProcDataCollector", File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.host.*.txt").Single())); + Assert.Contains("[coverlet]Mapping resolved", dataCollectorLogContent); + + // Process exits hang on clean seem that process doesn't close, maybe some msbuild node reuse? btw manually tested + // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath); + // Assert.False(File.Exists(sourceRootMappingFilePath)); + RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); + } - // Process exits hang on clean seem that process doesn't close, maybe some msbuild node reuse? btw manually tested - // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath); - // Assert.False(File.Exists(sourceRootMappingFilePath)); - RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); - } + private string GetReportPath(string standardOutput) + { + string reportPath = ""; + if (standardOutput.Contains("coverage.json")) + { +#pragma warning disable CS8602 // Dereference of a possibly null reference. + reportPath = standardOutput.Split('\n').FirstOrDefault(line => line.Contains("coverage.json")).TrimStart(); +#pragma warning restore CS8602 // Dereference of a possibly null reference. + reportPath = reportPath[reportPath.IndexOf(Directory.GetDirectoryRoot(_testProjectPath))..]; + reportPath = reportPath[..reportPath.IndexOf("coverage.json")]; + } + return reportPath; + } - public void Dispose() + public void Dispose() { File.Delete(Path.Combine(_testProjectPath, PropsFileName)); }