Skip to content

Commit fd0b3e7

Browse files
jalauzon-msftjalauzon-msft
authored
[Storage][DataMovement] Add perf tests for DMLib Track1 (#46768)
1 parent 3db918d commit fd0b3e7

File tree

14 files changed

+357
-22
lines changed

14 files changed

+357
-22
lines changed

‎eng/Packages.Data.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,7 @@
313313
<PackageReference Update="Microsoft.Azure.ResourceManager" Version="[1.1.0-preview]" />
314314
<PackageReference Update="Microsoft.Azure.Services.AppAuthentication" Version="[1.0.3, 2.0.0)" />
315315
<PackageReference Update="Microsoft.Azure.Storage.Blob" Version="11.1.7" />
316+
<PackageReference Update="Microsoft.Azure.Storage.DataMovement" Version="2.0.5" />
316317
<PackageReference Update="Microsoft.Azure.Storage.File" Version="11.2.2" />
317318
<PackageReference Update="Microsoft.Azure.Storage.Queue" Version="11.1.7" />
318319
<PackageReference Update="Microsoft.Azure.Test.HttpRecorder" Version="[1.13.3, 2.0.0)" />

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Azure.Storage.DataMovement.Blobs.Perf.csproj

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,4 @@
88
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj" />
99
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj" />
1010
</ItemGroup>
11-
12-
<ItemGroup>
13-
<Compile Include="$(AzureStorageSharedTestSources)\RepeatingStream.cs" LinkBase="Shared" />
14-
</ItemGroup>
1511
</Project>

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/DirectoryTransferTest.cs

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,26 +7,33 @@
77
using System.Threading;
88
using System.Threading.Tasks;
99
using Azure.Storage.Blobs;
10-
using Azure.Storage.Tests.Shared;
1110
using Azure.Test.Perf;
1211

1312
namespace Azure.Storage.DataMovement.Blobs.Perf
1413
{
1514
public abstract class DirectoryTransferTest<TOptions> : PerfTest<TOptions> where TOptions : DirectoryTransferOptions
1615
{
17-
protected Random Random { get; }
1816
protected BlobServiceClient BlobServiceClient { get; }
1917
protected LocalFilesStorageResourceProvider LocalFileResourceProvider { get; }
2018
protected BlobsStorageResourceProvider BlobResourceProvider { get; }
19+
20+
private TransferManager _transferManager;
2121
private TimeSpan _transferTimeout;
2222

2323
public DirectoryTransferTest(TOptions options) : base(options)
2424
{
25-
Random = new Random();
26-
BlobServiceClient = new BlobServiceClient(PerfTestEnvironment.Instance.BlobStorageEndpoint, PerfTestEnvironment.Instance.Credential);
25+
BlobServiceClient = new BlobServiceClient(PerfTestEnvironment.Instance.StorageEndpoint, PerfTestEnvironment.Instance.Credential);
2726
LocalFileResourceProvider = new LocalFilesStorageResourceProvider();
2827
BlobResourceProvider = new BlobsStorageResourceProvider(PerfTestEnvironment.Instance.Credential);
29-
_transferTimeout = TimeSpan.FromSeconds(5 + (Options.Count * Options.Size) / (1 * 1024 * 1024));
28+
_transferTimeout = TimeSpan.FromSeconds(10 + (Options.Count * Options.Size) / (1 * 1024 * 1024));
29+
30+
TransferManagerOptions managerOptions = new()
31+
{
32+
ErrorHandling = DataTransferErrorMode.StopOnAnyFailure,
33+
CheckpointerOptions = Options.DisableCheckpointer ? TransferCheckpointStoreOptions.Disabled() : default,
34+
MaximumConcurrency = Options.Concurrency
35+
};
36+
_transferManager = new TransferManager(managerOptions);
3037
}
3138

3239
protected string CreateLocalDirectory(bool populate = false)
@@ -39,7 +46,7 @@ protected string CreateLocalDirectory(bool populate = false)
3946
foreach (int i in Enumerable.Range(0, Options.Count))
4047
{
4148
string filePath = Path.Combine(directory, $"file{i}");
42-
using (RepeatingStream stream = new(1024 * 1024, Options.Size, true))
49+
using (Stream stream = RandomStream.Create(Options.Size))
4350
using (FileStream file = File.Open(filePath, FileMode.Create))
4451
{
4552
stream.CopyTo(file);
@@ -61,7 +68,7 @@ protected async Task<BlobContainerClient> CreateBlobContainerAsync(bool populate
6168
foreach (int i in Enumerable.Range(0, Options.Count))
6269
{
6370
BlobClient blob = container.GetBlobClient($"blob{i}");
64-
using (RepeatingStream stream = new(1024 * 1024, Options.Size, true))
71+
using (Stream stream = RandomStream.Create(Options.Size))
6572
{
6673
await blob.UploadAsync(stream);
6774
}
@@ -76,21 +83,14 @@ protected async Task RunAndVerifyTransferAsync(
7683
StorageResource destination,
7784
CancellationToken cancellationToken)
7885
{
79-
TransferManagerOptions managerOptions = new()
80-
{
81-
ErrorHandling = DataTransferErrorMode.StopOnAnyFailure,
82-
CheckpointerOptions = Options.DisableCheckpointer ? TransferCheckpointStoreOptions.Disabled() : default
83-
};
84-
TransferManager transferManager = new(managerOptions);
85-
8686
DataTransferOptions options = new()
8787
{
8888
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
8989
InitialTransferSize = Options.InitialTransferSize,
9090
MaximumTransferChunkSize = Options.ChunkSize,
9191
};
9292
options.ItemTransferFailed += HandleFailure;
93-
DataTransfer transfer = await transferManager.StartTransferAsync(
93+
DataTransfer transfer = await _transferManager.StartTransferAsync(
9494
source, destination, options, cancellationToken);
9595

9696
// The test runs for a specified duration and then cancels the token.

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/PerfTestEnvironment.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,19 +25,19 @@ internal sealed class PerfTestEnvironment : TestEnvironment
2525
/// The name of the Blob storage account to test against.
2626
/// </summary>
2727
/// <value>The Blob storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value>
28-
public string BlobStorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
28+
public string StorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
2929

3030
/// <summary>
3131
/// The Blob storage endpoint.
3232
/// </summary>
33-
public Uri BlobStorageEndpoint { get; }
33+
public Uri StorageEndpoint { get; }
3434

3535
/// <summary>
3636
/// Initializes a new instance of the <see cref="PerfTestEnvironment"/> class.
3737
/// </summary>
3838
public PerfTestEnvironment()
3939
{
40-
BlobStorageEndpoint = new Uri($"https://{BlobStorageAccountName}.blob.{StorageEndpointSuffix}");
40+
StorageEndpoint = new Uri($"https://{StorageAccountName}.blob.{StorageEndpointSuffix}");
4141
}
4242
}
4343
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Options/DirectoryTransferOptions.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ public class DirectoryTransferOptions : PerfOptions
2020
[Option("chunk-size", HelpText = "The chunk/block size to use during transfers (in bytes)")]
2121
public long? ChunkSize { get; set; }
2222

23+
[Option("concurrency", HelpText = "The max concurrency to use during each transfer.")]
24+
public int? Concurrency { get; set; }
25+
2326
[Option("disable-checkpointer", HelpText = "Set to disable checkpointing.")]
2427
public bool DisableCheckpointer { get; set; }
2528

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Infrastructure/DirectoryTransferTest.cs

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.IO;
6+
using System.Linq;
7+
using System.Threading.Tasks;
8+
using Azure.Test.Perf;
9+
using Microsoft.Azure.Storage.Auth;
10+
using Microsoft.Azure.Storage.Blob;
11+
12+
namespace Microsoft.Azure.Storage.DataMovement.Perf
13+
{
14+
public abstract class DirectoryTransferTest<TOptions> : PerfTest<TOptions> where TOptions : DirectoryTransferOptions
15+
{
16+
protected CloudBlobClient BlobClient;
17+
18+
protected static DirectoryTransferContext DefaultTransferContext => new()
19+
{
20+
ShouldOverwriteCallbackAsync = DirectoryTransferContext.ForceOverwrite
21+
};
22+
23+
public DirectoryTransferTest(TOptions options) : base(options)
24+
{
25+
StorageCredentials credentials = new(
26+
PerfTestEnvironment.Instance.StorageAccountName,
27+
PerfTestEnvironment.Instance.StorageAccountKey);
28+
29+
CloudStorageAccount account = new(credentials, PerfTestEnvironment.Instance.StorageEndpointSuffix, useHttps: true);
30+
BlobClient = account.CreateCloudBlobClient();
31+
32+
if (Options.ChunkSize.HasValue)
33+
{
34+
TransferManager.Configurations.BlockSize = (int)Options.ChunkSize.Value;
35+
}
36+
if (Options.Concurrency.HasValue)
37+
{
38+
TransferManager.Configurations.ParallelOperations = Options.Concurrency.Value;
39+
}
40+
}
41+
42+
protected string CreateLocalDirectory(bool populate = false)
43+
{
44+
string directory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
45+
Directory.CreateDirectory(directory);
46+
47+
if (populate)
48+
{
49+
foreach (int i in Enumerable.Range(0, Options.Count))
50+
{
51+
string filePath = Path.Combine(directory, $"file{i}");
52+
using (Stream stream = RandomStream.Create(Options.Size))
53+
using (FileStream file = System.IO.File.Open(filePath, FileMode.Create))
54+
{
55+
stream.CopyTo(file);
56+
}
57+
}
58+
}
59+
60+
return directory;
61+
}
62+
63+
protected async Task<CloudBlobContainer> CreateBlobContainerAsync(bool populate = false)
64+
{
65+
string containerName = $"test-{Guid.NewGuid()}".ToLowerInvariant();
66+
CloudBlobContainer container = BlobClient.GetContainerReference(containerName);
67+
await container.CreateIfNotExistsAsync();
68+
69+
if (populate)
70+
{
71+
foreach (int i in Enumerable.Range(0, Options.Count))
72+
{
73+
CloudBlockBlob blob = container.GetBlockBlobReference($"blob{i}");
74+
using (Stream stream = RandomStream.Create(Options.Size))
75+
{
76+
await blob.UploadFromStreamAsync(stream);
77+
}
78+
}
79+
}
80+
81+
return container;
82+
}
83+
84+
protected void AssertTransferStatus(TransferStatus status)
85+
{
86+
if (status.NumberOfFilesSkipped > 0)
87+
{
88+
throw new Exception($"Transfer contained {status.NumberOfFilesSkipped} skipped files.");
89+
}
90+
if (status.NumberOfFilesFailed > 0)
91+
{
92+
throw new Exception($"Transfer contaiend {status.NumberOfFilesFailed} failed files.");
93+
}
94+
}
95+
}
96+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Infrastructure/PerfTestEnvironment.cs

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Core.TestFramework;
5+
6+
namespace Microsoft.Azure.Storage.DataMovement.Perf
7+
{
8+
/// <summary>
9+
/// Represents the ambient environment in which the test suite is being run, offering access to information such as environment variables.
10+
/// </summary>
11+
internal sealed class PerfTestEnvironment : TestEnvironment
12+
{
13+
/// <summary>
14+
/// The shared instance of the <see cref="PerfTestEnvironment"/> to be used during test runs.
15+
/// </summary>
16+
public static PerfTestEnvironment Instance { get; } = new PerfTestEnvironment();
17+
18+
/// <summary>
19+
/// The storage account endpoint suffix to use for testing.
20+
/// </summary>
21+
public new string StorageEndpointSuffix => base.StorageEndpointSuffix ?? "core.windows.net";
22+
23+
/// <summary>
24+
/// The name of the Blob storage account to test against.
25+
/// </summary>
26+
/// <value>The Blob storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value>
27+
public string StorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
28+
29+
/// <summary>
30+
/// The shared access key of the Blob storage account to test against.
31+
/// </summary>
32+
/// <value>The Blob storage account key, read from the "AZURE_STORAGE_ACCOUNT_KEY" environment variable.</value>
33+
public string StorageAccountKey => GetVariable("AZURE_STORAGE_ACCOUNT_KEY");
34+
}
35+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Microsoft.Azure.Storage.DataMovement.Perf.csproj

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<OutputType>Exe</OutputType>
4+
</PropertyGroup>
5+
<ItemGroup>
6+
<PackageReference Include="Microsoft.Azure.Storage.DataMovement" />
7+
</ItemGroup>
8+
9+
<ItemGroup>
10+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj" />
11+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj" />
12+
</ItemGroup>
13+
</Project>

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Options/DirectoryTransferOptions.cs

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Test.Perf;
5+
using CommandLine;
6+
7+
namespace Microsoft.Azure.Storage.DataMovement.Perf
8+
{
9+
public class DirectoryTransferOptions : PerfOptions
10+
{
11+
[Option('c', "count", Default = 10, HelpText = "Number of items in each transfer.")]
12+
public int Count { get; set; }
13+
14+
[Option('s', "size", Default = 1024, HelpText = "Size of each file (in bytes)")]
15+
public long Size { get; set; }
16+
17+
[Option("chunk-size", HelpText = "The chunk/block size to use during transfers (in bytes)")]
18+
public long? ChunkSize { get; set; }
19+
20+
[Option("concurrency", HelpText = "The max concurrency to use during each transfer.")]
21+
public int? Concurrency { get; set; }
22+
23+
// Override warmup to set default to 0
24+
[Option('w', "warmup", Default = 0, HelpText = "Duration of warmup in seconds")]
25+
public new int Warmup { get; set; }
26+
}
27+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Program.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.Reflection;
5+
using Azure.Test.Perf;
6+
7+
await PerfProgram.Main(Assembly.GetEntryAssembly(), args);

0 commit comments

Comments
 (0)