Skip to content

Commit 46a198a

Browse files
kotlarmilosmatouskozak
kotlarmilos
and
matouskozak
authored
Add build time measurement to Startup tool (#4842)
* Download binlog files * Add devicebuild scenario and implement build time parser in the Startup tool * Copy binlog files to payload directory * Fix android binlog path * Update BuildTimeParser to measure total build time * Fix binlog path * Fix android binlog path * Switch the default counter settings in BuildTimeParser * Measure AndroidAppBuilderTask * Update scripts/run_performance_job.py Co-authored-by: Matous Kozak <[email protected]> * Rename DEVICEBUILD to BUILDTIME * Check for binlog path * Improve file handling --------- Co-authored-by: Matous Kozak <[email protected]>
1 parent b4b252f commit 46a198a

File tree

9 files changed

+165
-6
lines changed

9 files changed

+165
-6
lines changed

‎eng/pipelines/templates/runtime-perf-job.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,27 @@ jobs:
180180

181181
displayName: 'Android Sample App'
182182

183+
- task: DownloadBuildArtifacts@0
184+
displayName: 'Download binlog files'
185+
inputs:
186+
buildType: current
187+
downloadType: single
188+
downloadPath: '$(builtAppDir)/androidHelloWorldBinlog'
189+
# AndroidMono
190+
${{ if eq(parameters.runtimeType, 'AndroidMono')}}:
191+
${{ if eq(parameters.codeGenType, 'JIT') }}:
192+
artifactName: 'AndroidMonoArm64BuildLog'
193+
${{ if eq(parameters.codeGenType, 'AOT') }}:
194+
artifactName: 'AndroidMonoAOTArm64BuildLog'
195+
196+
# AndroidCoreCLR
197+
${{ if eq(parameters.runtimeType, 'AndroidCoreCLR')}}:
198+
${{ if eq(parameters.codeGenType, 'JIT') }}:
199+
artifactName: 'AndroidCoreCLRArm64BuildLog'
200+
${{ if eq(parameters.codeGenType, 'R2R') }}:
201+
artifactName: 'AndroidCoreCLRR2RArm64BuildLog'
202+
checkDownloadedFiles: true
203+
183204
# Disabled due to not working and needing consistent normal android results. https://github.com/dotnet/performance/issues/4729
184205
# - template: /eng/pipelines/templates/download-artifact-step.yml
185206
# parameters:
@@ -233,6 +254,25 @@ jobs:
233254
${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'True')) }}:
234255
artifactName: 'iOSSampleAppNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}'
235256
checkDownloadedFiles: true
257+
- task: DownloadBuildArtifacts@0
258+
displayName: 'Download binlog files'
259+
inputs:
260+
buildType: current
261+
downloadType: single
262+
downloadPath: '$(builtAppDir)/iosHelloWorldBinlog'
263+
${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'False')) }}:
264+
artifactName: 'iOSMonoArm64NoLLVMNoStripSymbolsBuildLog'
265+
${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'True')) }}:
266+
artifactName: 'iOSMonoArm64NoLLVMStripSymbolsBuildLog'
267+
${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'False')) }}:
268+
artifactName: 'iOSMonoArm64LLVMNoStripSymbolsBuildLog'
269+
${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'True')) }}:
270+
artifactName: 'iOSMonoArm64LLVMStripSymbolsBuildLog'
271+
${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'False')) }}:
272+
artifactName: 'iOSNativeAOTArm64NoStripSymbolsBuildLog'
273+
${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'True')) }}:
274+
artifactName: 'iOSNativeAOTArm64StripSymbolsBuildLog'
275+
checkDownloadedFiles: true
236276

237277
- ${{ if notIn(parameters.runtimeType, 'wasm', 'AndroidMono', 'AndroidCoreCLR', 'iOSMono', 'iOSNativeAOT') }}:
238278
- ${{ if ne(parameters.runtimeFlavor, 'Mono') }}:

‎scripts/run_performance_job.py‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -738,6 +738,15 @@ def run_performance_job(args: RunPerformanceJobArgs):
738738
raise Exception("Built apps directory must be present for Android benchmarks")
739739
getLogger().info("Copying Android apps to payload directory")
740740
shutil.copy(os.path.join(args.built_app_dir, "androidHelloWorld", "HelloAndroid.apk"), os.path.join(root_payload_dir, "HelloAndroid.apk"))
741+
742+
android_binlog_dir = os.path.join(root_payload_dir, "androidHelloWorldBinlog")
743+
shutil.copytree(os.path.join(args.built_app_dir, "androidHelloWorldBinlog"), android_binlog_dir)
744+
745+
binlog_files = glob(os.path.join(android_binlog_dir, "**", "*.binlog"))
746+
if binlog_files:
747+
dest = os.path.join(android_binlog_dir, "msbuild.binlog")
748+
getLogger().info(f"Moving {binlog_files[0]} to {dest}")
749+
shutil.move(binlog_files[0], dest)
741750
# Disabled due to not successfully building at the moment. https://github.com/dotnet/performance/issues/4729
742751
# if android_mono:
743752
# shutil.copy(os.path.join(args.built_app_dir, "MonoBenchmarksDroid.apk"), os.path.join(root_payload_dir, "MonoBenchmarksDroid.apk"))
@@ -756,11 +765,20 @@ def run_performance_job(args: RunPerformanceJobArgs):
756765
shutil.copytree(os.path.join(args.built_app_dir, "iosHelloWorldZip"), ios_hello_world_zip_dir)
757766

758767
# Find the zip file in the directory and move it to iOSSampleApp.zip
759-
for file in glob(os.path.join(ios_hello_world_zip_dir, "**", "*.zip")):
768+
zip_files = glob(os.path.join(ios_hello_world_zip_dir, "**", "*.zip"))
769+
if zip_files:
760770
dest = os.path.join(ios_hello_world_zip_dir, "iOSSampleApp.zip")
761-
getLogger().info(f"Moving {file} to {dest}")
762-
shutil.move(file, dest)
763-
break
771+
getLogger().info(f"Moving {zip_files[0]} to {dest}")
772+
shutil.move(zip_files[0], dest)
773+
774+
ios_hello_world_binlog_dir = os.path.join(payload_dir, "iosHelloWorldBinlog")
775+
shutil.copytree(os.path.join(args.built_app_dir, "iosHelloWorldBinlog"), ios_hello_world_binlog_dir)
776+
777+
binlog_files = glob(os.path.join(ios_hello_world_binlog_dir, "**", "*.binlog"))
778+
if binlog_files:
779+
dest = os.path.join(ios_hello_world_binlog_dir, "msbuild.binlog")
780+
getLogger().info(f"Moving {binlog_files[0]} to {dest}")
781+
shutil.move(binlog_files[0], dest)
764782

765783
# ensure work item directory is not empty
766784
getLogger().info("Copying docs to work item directory so it isn't empty")

‎src/scenarios/shared/const.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,16 @@
1818
DEVICEMEMORYCONSUMPTION = "devicememoryconsumption"
1919
ANDROIDINSTRUMENTATION = "androidinstrumentation"
2020
DEVICEPOWERCONSUMPTION = "devicepowerconsumption"
21+
BUILDTIME = "buildtime"
2122

2223
SCENARIO_NAMES = {STARTUP: 'Startup',
2324
SDK: 'SDK',
2425
CROSSGEN: 'Crossgen',
2526
CROSSGEN2: 'Crossgen2',
2627
INNERLOOP: 'Innerloop',
2728
INNERLOOPMSBUILD: 'InnerLoopMsBuild',
28-
DOTNETWATCH: 'DotnetWatch'}
29+
DOTNETWATCH: 'DotnetWatch',
30+
BUILDTIME: 'BuildTime'}
2931

3032
BINDIR = 'bin'
3133
PUBDIR = 'pub'

‎src/scenarios/shared/runner.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ def __init__(self, traits: TestTraits):
4747
self.crossgen_arguments = CrossgenArguments()
4848
self.affinity = None
4949
self.upload_to_perflab_container = False
50+
self.binlogpath = None
5051
setup_loggers(True)
5152

5253
def parseargs(self):
@@ -168,6 +169,11 @@ def parseargs(self):
168169
''' )
169170
self.add_common_arguments(sodparser)
170171

172+
buildtimeparser = subparsers.add_parser(const.BUILDTIME,
173+
description='measure build time from a binlog')
174+
buildtimeparser.add_argument('--binlog-path', help='Location of binlog', dest='binlogpath')
175+
self.add_common_arguments(buildtimeparser)
176+
171177
args = parser.parse_args()
172178

173179
if not args.testtype:
@@ -187,6 +193,9 @@ def parseargs(self):
187193

188194
if self.testtype == const.SOD:
189195
self.dirs = args.dirs
196+
197+
if self.testtype == const.BUILDTIME:
198+
self.binlogpath = args.binlogpath
190199

191200
if self.testtype == const.DEVICESTARTUP:
192201
self.packagepath = args.packagepath
@@ -957,3 +966,10 @@ def run(self):
957966
if not (self.dirs or builtdir):
958967
raise Exception("Dirs was not passed in and neither %s nor %s exist" % (const.PUBDIR, const.BINDIR))
959968
sod.runtests(scenarioname=self.scenarioname, dirs=self.dirs or builtdir, upload_to_perflab_container=self.upload_to_perflab_container, artifact=self.traits.artifact)
969+
970+
elif self.testtype == const.BUILDTIME:
971+
startup = StartupWrapper()
972+
if not (self.binlogpath and os.path.exists(self.binlogpath)):
973+
raise Exception("For build time measurements a valid binlog path must be provided.")
974+
self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.BUILDTIME, tracename=self.binlogpath, scenarioname=self.scenarioname)
975+
startup.parsetraces(self.traits)

‎src/scenarios/shared/testtraits.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
const.CROSSGEN2,
99
const.SOD,
1010
const.INNERLOOP,
11-
const.DEVICESTARTUP]
11+
const.DEVICESTARTUP,
12+
const.BUILDTIME]
1213

1314
class TestTraits:
1415

‎src/tools/ScenarioMeasurement/Directory.Packages.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,6 @@
99
<PackageVersion Include="coverlet.collector" Version="1.0.1" />
1010
<PackageVersion Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="3.1.7" />
1111
<PackageVersion Include="System.Security.Principal.Windows" Version="4.7.0" />
12+
<PackageVersion Include="MSBuild.StructuredLogger" Version="2.2.386" />
1213
</ItemGroup>
1314
</Project>

‎src/tools/ScenarioMeasurement/Startup/Startup.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ enum MetricType
2525
WinUI,
2626
WinUIBlazor,
2727
TimeToMain2,
28+
BuildTime,
2829
}
2930

3031
public class InnerLoopMarkerEventSource : EventSource
@@ -288,6 +289,7 @@ static void checkArg(string arg, string name)
288289
MetricType.WinUI => new WinUIParser(),
289290
MetricType.WinUIBlazor => new WinUIBlazorParser(),
290291
MetricType.TimeToMain2 => new TimeToMain2Parser(AddTestProcessEnvironmentVariable),
292+
MetricType.BuildTime => new BuildTimeParser(),
291293
_ => throw new ArgumentOutOfRangeException(),
292294
};
293295

‎src/tools/ScenarioMeasurement/Util/Parsers/BuildTimeParser.cs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using Microsoft.Build.Logging.StructuredLogger;
6+
using StructuredLogViewer;
7+
using Microsoft.Diagnostics.Tracing;
8+
using Reporting;
9+
10+
namespace ScenarioMeasurement;
11+
12+
/// <summary>
13+
/// Parses the build time from a binary log file.
14+
/// </summary>
15+
public class BuildTimeParser : IParser
16+
{
17+
public void EnableKernelProvider(ITraceSession kernel)
18+
{
19+
throw new NotImplementedException();
20+
}
21+
22+
public void EnableUserProviders(ITraceSession user)
23+
{
24+
throw new NotImplementedException();
25+
}
26+
27+
public IEnumerable<Counter> Parse(string binlogFile, string processName, IList<int> pids, string commandLine)
28+
{
29+
var illinkTimes = new List<double>();
30+
var monoaotcompilerTimes = new List<double>();
31+
var appleappbuilderTimes = new List<double>();
32+
var androidappbuilderTimes = new List<double>();
33+
var publishTimes = new List<double>();
34+
35+
if (File.Exists(binlogFile))
36+
{
37+
var build = BinaryLog.ReadBuild(binlogFile);
38+
BuildAnalyzer.AnalyzeBuild(build);
39+
40+
foreach (var task in build.FindChildrenRecursive<Task>())
41+
{
42+
var name = task.Name;
43+
var ms = task.Duration.TotalMilliseconds;
44+
45+
if (name.Equals("ILLink", StringComparison.OrdinalIgnoreCase))
46+
{
47+
illinkTimes.Add(ms);
48+
}
49+
else if (name.Equals("MonoAOTCompiler", StringComparison.OrdinalIgnoreCase))
50+
{
51+
monoaotcompilerTimes.Add(ms);
52+
}
53+
else if (name.Equals("AppleAppBuilderTask", StringComparison.OrdinalIgnoreCase))
54+
{
55+
appleappbuilderTimes.Add(ms);
56+
}
57+
else if (name.Equals("AndroidAppBuilderTask", StringComparison.OrdinalIgnoreCase))
58+
{
59+
androidappbuilderTimes.Add(ms);
60+
}
61+
}
62+
63+
publishTimes.Add(build.Duration.TotalMilliseconds);
64+
}
65+
66+
67+
if (illinkTimes.Count > 0)
68+
yield return new Counter { Name = "ILLink Time", MetricName = "ms", DefaultCounter = false, TopCounter = true, Results = illinkTimes.ToArray() };
69+
if (monoaotcompilerTimes.Count > 0)
70+
yield return new Counter { Name = "MonoAOTCompiler Time", MetricName = "ms", DefaultCounter = false, TopCounter = true, Results = monoaotcompilerTimes.ToArray() };
71+
if (appleappbuilderTimes.Count > 0)
72+
yield return new Counter { Name = "AppleAppBuilderTask Time", MetricName = "ms", DefaultCounter = false, TopCounter = true, Results = appleappbuilderTimes.ToArray() };
73+
if (androidappbuilderTimes.Count > 0)
74+
yield return new Counter { Name = "AndroidAppBuilderTask Time", MetricName = "ms", DefaultCounter = false, TopCounter = true, Results = androidappbuilderTimes.ToArray() };
75+
if (publishTimes.Count > 0)
76+
yield return new Counter { Name = "Publish Time", MetricName = "ms", DefaultCounter = true, TopCounter = true, Results = publishTimes.ToArray() };
77+
}
78+
}

‎src/tools/ScenarioMeasurement/Util/Util.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
<ItemGroup>
1515
<PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" />
1616
<PackageReference Include="System.Security.Principal.Windows" />
17+
<PackageReference Include="MSBuild.StructuredLogger" />
1718
</ItemGroup>
1819

1920
<ItemGroup>

0 commit comments

Comments
 (0)