Add failover & load balancing telemetry (#586)

zhiyuanliang-ms · amerjusupovic · web-flow · commit d5515369a5ef · 2024-08-20T14:41:09.000+08:00
* add failover telemetry

* add load balancing enabled tracing

* use Features=xxx+xxx pattern

* revert change

* add comment

* update context string &amp; move helper method to TracingUtil

* add comments

* update

* revert change

* add checks

* remove unused package

* fix typo

---------

Co-authored-by: Amer Jusupovic &lt;32405726+amerjusupovic@users.noreply.github.com&gt;
diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs
@@ -970,7 +970,8 @@ private void SetRequestTracingOptions()
                 IsKeyVaultConfigured = _options.IsKeyVaultConfigured,
                 IsKeyVaultRefreshConfigured = _options.IsKeyVaultRefreshConfigured,
                 ReplicaCount = _options.Endpoints?.Count() - 1 ?? _options.ConnectionStrings?.Count() - 1 ?? 0,
-                FeatureFlagTracing = _options.FeatureFlagTracing
+                FeatureFlagTracing = _options.FeatureFlagTracing,
+                IsLoadBalancingEnabled = _options.LoadBalancingEnabled
             };
         }
 
@@ -1005,6 +1006,11 @@ private async Task<T> ExecuteWithFailOverPolicyAsync<T>(
             Func<ConfigurationClient, Task<T>> funcToExecute,
             CancellationToken cancellationToken = default)
         {
+            if (_requestTracingEnabled && _requestTracingOptions != null)
+            {
+                _requestTracingOptions.IsFailoverRequest = false;
+            }
+
             if (_options.LoadBalancingEnabled && _lastSuccessfulEndpoint != null && clients.Count() > 1)
             {
                 int nextClientIndex = 0;
@@ -1107,6 +1113,11 @@ private async Task<T> ExecuteWithFailOverPolicyAsync<T>(
                 }
 
                 previousEndpoint = currentEndpoint;
+
+                if (_requestTracingEnabled && _requestTracingOptions != null)
+                {
+                    _requestTracingOptions.IsFailoverRequest = true;
+                }
             }
         }
 
diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs
@@ -22,20 +22,28 @@ internal class RequestTracingConstants
 
         public const string RequestTypeKey = "RequestType";
         public const string HostTypeKey = "Host";
-        public const string FilterTypeKey = "Filter";
         public const string EnvironmentKey = "Env";
         public const string FeatureManagementVersionKey = "FMVer";
         public const string FeatureManagementAspNetCoreVersionKey = "FMANCVer";
         public const string DevEnvironmentValue = "Dev";
         public const string KeyVaultConfiguredTag = "UsesKeyVault";
         public const string KeyVaultRefreshConfiguredTag = "RefreshesKeyVault";
         public const string ReplicaCountKey = "ReplicaCount";
+        public const string FeaturesKey = "Features";
+        public const string LoadBalancingEnabledTag = "LB";
+        public const string SignalRUsedTag = "SignalR";
+        public const string FailoverRequestTag = "Failover";
+
+        public const string FeatureFlagFilterTypeKey = "Filter";
+        public const string CustomFilter = "CSTM";
+        public const string PercentageFilter = "PRCNT";
+        public const string TimeWindowFilter = "TIME";
+        public const string TargetingFilter = "TRGT";
+        public const string FeatureFlagFeaturesKey = "FFFeatures";
         public const string FeatureFlagUsesTelemetryTag = "Telemetry";
         public const string FeatureFlagUsesSeedTag = "Seed";
         public const string FeatureFlagMaxVariantsKey = "MaxVariants";
         public const string FeatureFlagUsesVariantConfigurationReferenceTag = "ConfigRef";
-        public const string FeatureFlagFeaturesKey = "FFFeatures";
-        public const string SignalRUsedTag = "UsesSignalR";
 
         public const string DiagnosticHeaderActivityName = "Azure.CustomDiagnosticHeaders";
         public const string CorrelationContextHeader = "Correlation-Context";
diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs
@@ -14,11 +14,6 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManage
     /// </summary>
     internal class FeatureFlagTracing
     {
-        private const string CustomFilter = "CSTM";
-        private const string PercentageFilter = "PRCNT";
-        private const string TimeWindowFilter = "TIME";
-        private const string TargetingFilter = "TRGT";
-
         // Built-in Feature Filter Names
         private readonly List<string> PercentageFilterNames = new List<string> { "Percentage", "Microsoft.Percentage", "PercentageFilter", "Microsoft.PercentageFilter" };
         private readonly List<string> TimeWindowFilterNames = new List<string> { "TimeWindow", "Microsoft.TimeWindow", "TimeWindowFilter", "Microsoft.TimeWindowFilter" };
@@ -98,7 +93,7 @@ public string CreateFiltersString()
 
             if (UsesCustomFilter)
             {
-                sb.Append(CustomFilter);
+                sb.Append(RequestTracingConstants.CustomFilter);
             }
 
             if (UsesPercentageFilter)
@@ -108,7 +103,7 @@ public string CreateFiltersString()
                     sb.Append(RequestTracingConstants.Delimiter);
                 }
 
-                sb.Append(PercentageFilter);
+                sb.Append(RequestTracingConstants.PercentageFilter);
             }
 
             if (UsesTimeWindowFilter)
@@ -118,7 +113,7 @@ public string CreateFiltersString()
                     sb.Append(RequestTracingConstants.Delimiter);
                 }
 
-                sb.Append(TimeWindowFilter);
+                sb.Append(RequestTracingConstants.TimeWindowFilter);
             }
 
             if (UsesTargetingFilter)
@@ -128,14 +123,23 @@ public string CreateFiltersString()
                     sb.Append(RequestTracingConstants.Delimiter);
                 }
 
-                sb.Append(TargetingFilter);
+                sb.Append(RequestTracingConstants.TargetingFilter);
             }
 
             return sb.ToString();
         }
 
+        /// <summary>
+        /// Returns a formatted string containing code names, indicating which tracing features are used by feature flags.
+        /// </summary>
+        /// <returns>Formatted string like: "Seed+ConfigRef+Telemetry". If no tracing features are used, empty string will be returned.</returns>
         public string CreateFeaturesString()
         {
+            if (!UsesAnyTracingFeature())
+            {
+                return string.Empty;
+            }
+
             var sb = new StringBuilder();
 
             if (UsesSeed)
diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs
@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 //
 using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement;
+using System.Text;
 
 namespace Microsoft.Extensions.Configuration.AzureAppConfiguration
 {
@@ -51,5 +52,55 @@ internal class RequestTracingOptions
         /// Flag to indicate whether Microsoft.AspNetCore.SignalR assembly is present in the application.
         /// </summary>
         public bool IsSignalRUsed { get; set; } = false;
+
+        /// <summary>
+        /// Flag to indicate whether load balancing is enabled.
+        /// </summary>
+        public bool IsLoadBalancingEnabled { get; set; } = false;
+
+        /// <summary>
+        /// Flag to indicate whether the request is triggered by a failover.
+        /// </summary>
+        public bool IsFailoverRequest { get; set; } = false;
+
+        /// <summary>
+        /// Checks whether any tracing feature is used.
+        /// </summary>
+        /// <returns>True if any tracing feature is used, otherwise false.</returns>
+        public bool UsesAnyTracingFeature()
+        {
+            return IsLoadBalancingEnabled || IsSignalRUsed;
+        }
+
+        /// <summary>
+        /// Returns a formatted string containing code names, indicating which tracing features are used by the application.
+        /// </summary>
+        /// <returns>Formatted string like: "LB+SignalR". If no tracing features are used, empty string will be returned.</returns>
+        public string CreateFeaturesString()
+        {
+            if (!UsesAnyTracingFeature())
+            {
+                return string.Empty;
+            }
+
+            var sb = new StringBuilder();
+
+            if (IsLoadBalancingEnabled)
+            {
+                sb.Append(RequestTracingConstants.LoadBalancingEnabledTag);
+            }
+
+            if (IsSignalRUsed)
+            {
+                if (sb.Length > 0)
+                {
+                    sb.Append(RequestTracingConstants.Delimiter);
+                }
+
+                sb.Append(RequestTracingConstants.SignalRUsedTag);
+            }
+
+            return sb.ToString();
+        }
     }
 }
diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/TracingUtils.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/TracingUtils.cs
@@ -158,7 +158,7 @@ private static string CreateCorrelationContextHeader(RequestType requestType, Re
 
             if (requestTracingOptions.FeatureFlagTracing.UsesAnyFeatureFilter())
             {
-                correlationContextKeyValues.Add(new KeyValuePair<string, string>(RequestTracingConstants.FilterTypeKey, requestTracingOptions.FeatureFlagTracing.CreateFiltersString()));
+                correlationContextKeyValues.Add(new KeyValuePair<string, string>(RequestTracingConstants.FeatureFlagFilterTypeKey, requestTracingOptions.FeatureFlagTracing.CreateFiltersString()));
             }
 
             if (requestTracingOptions.FeatureFlagTracing.MaxVariants > 0)
@@ -181,6 +181,11 @@ private static string CreateCorrelationContextHeader(RequestType requestType, Re
                 correlationContextKeyValues.Add(new KeyValuePair<string, string>(RequestTracingConstants.FeatureManagementAspNetCoreVersionKey, requestTracingOptions.FeatureManagementAspNetCoreVersion));
             }
 
+            if (requestTracingOptions.UsesAnyTracingFeature())
+            {
+                correlationContextKeyValues.Add(new KeyValuePair<string, string>(RequestTracingConstants.FeaturesKey, requestTracingOptions.CreateFeaturesString()));
+            }
+
             if (requestTracingOptions.IsKeyVaultConfigured)
             {
                 correlationContextTags.Add(RequestTracingConstants.KeyVaultConfiguredTag);
@@ -191,9 +196,9 @@ private static string CreateCorrelationContextHeader(RequestType requestType, Re
                 correlationContextTags.Add(RequestTracingConstants.KeyVaultRefreshConfiguredTag);
             }
 
-            if (requestTracingOptions.IsSignalRUsed)
+            if (requestTracingOptions.IsFailoverRequest)
             {
-                correlationContextTags.Add(RequestTracingConstants.SignalRUsedTag);
+                correlationContextTags.Add(RequestTracingConstants.FailoverRequestTag);
             }
 
             var sb = new StringBuilder();

Original file line number	Diff line number	Diff line change
`@@ -970,7 +970,8 @@ private void SetRequestTracingOptions()`
`970`	`970`	`IsKeyVaultConfigured = _options.IsKeyVaultConfigured,`
`971`	`971`	`IsKeyVaultRefreshConfigured = _options.IsKeyVaultRefreshConfigured,`
`972`	`972`	`ReplicaCount = _options.Endpoints?.Count() - 1 ?? _options.ConnectionStrings?.Count() - 1 ?? 0,`
`973`		`- FeatureFlagTracing = _options.FeatureFlagTracing`
	`973`	`+ FeatureFlagTracing = _options.FeatureFlagTracing,`
	`974`	`+ IsLoadBalancingEnabled = _options.LoadBalancingEnabled`
`974`	`975`	`};`
`975`	`976`	`}`
`976`	`977`
`@@ -1005,6 +1006,11 @@ private async Task<T> ExecuteWithFailOverPolicyAsync<T>(`
`1005`	`1006`	`Func<ConfigurationClient, Task<T>> funcToExecute,`
`1006`	`1007`	`CancellationToken cancellationToken = default)`
`1007`	`1008`	`{`
	`1009`	`+ if (_requestTracingEnabled && _requestTracingOptions != null)`
	`1010`	`+ {`
	`1011`	`+ _requestTracingOptions.IsFailoverRequest = false;`
	`1012`	`+ }`
	`1013`	`+`
`1008`	`1014`	`if (_options.LoadBalancingEnabled && _lastSuccessfulEndpoint != null && clients.Count() > 1)`
`1009`	`1015`	`{`
`1010`	`1016`	`int nextClientIndex = 0;`
`@@ -1107,6 +1113,11 @@ private async Task<T> ExecuteWithFailOverPolicyAsync<T>(`
`1107`	`1113`	`}`
`1108`	`1114`
`1109`	`1115`	`previousEndpoint = currentEndpoint;`
	`1116`	`+`
	`1117`	`+ if (_requestTracingEnabled && _requestTracingOptions != null)`
	`1118`	`+ {`
	`1119`	`+ _requestTracingOptions.IsFailoverRequest = true;`
	`1120`	`+ }`
`1110`	`1121`	`}`
`1111`	`1122`	`}`
`1112`	`1123`
Original file line number	Diff line number	Diff line change
`@@ -14,11 +14,6 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManage`
`14`	`14`	`/// </summary>`
`15`	`15`	`internal class FeatureFlagTracing`
`16`	`16`	`{`
`17`		`- private const string CustomFilter = "CSTM";`
`18`		`- private const string PercentageFilter = "PRCNT";`
`19`		`- private const string TimeWindowFilter = "TIME";`
`20`		`- private const string TargetingFilter = "TRGT";`
`21`		`-`
`22`	`17`	`// Built-in Feature Filter Names`
`23`	`18`	`private readonly List<string> PercentageFilterNames = new List<string> { "Percentage", "Microsoft.Percentage", "PercentageFilter", "Microsoft.PercentageFilter" };`
`24`	`19`	`private readonly List<string> TimeWindowFilterNames = new List<string> { "TimeWindow", "Microsoft.TimeWindow", "TimeWindowFilter", "Microsoft.TimeWindowFilter" };`
`@@ -98,7 +93,7 @@ public string CreateFiltersString()`
`98`	`93`
`99`	`94`	`if (UsesCustomFilter)`
`100`	`95`	`{`
`101`		`- sb.Append(CustomFilter);`
	`96`	`+ sb.Append(RequestTracingConstants.CustomFilter);`
`102`	`97`	`}`
`103`	`98`
`104`	`99`	`if (UsesPercentageFilter)`
`@@ -108,7 +103,7 @@ public string CreateFiltersString()`
`108`	`103`	`sb.Append(RequestTracingConstants.Delimiter);`
`109`	`104`	`}`
`110`	`105`
`111`		`- sb.Append(PercentageFilter);`
	`106`	`+ sb.Append(RequestTracingConstants.PercentageFilter);`
`112`	`107`	`}`
`113`	`108`
`114`	`109`	`if (UsesTimeWindowFilter)`
`@@ -118,7 +113,7 @@ public string CreateFiltersString()`
`118`	`113`	`sb.Append(RequestTracingConstants.Delimiter);`
`119`	`114`	`}`
`120`	`115`
`121`		`- sb.Append(TimeWindowFilter);`
	`116`	`+ sb.Append(RequestTracingConstants.TimeWindowFilter);`
`122`	`117`	`}`
`123`	`118`
`124`	`119`	`if (UsesTargetingFilter)`
`@@ -128,14 +123,23 @@ public string CreateFiltersString()`
`128`	`123`	`sb.Append(RequestTracingConstants.Delimiter);`
`129`	`124`	`}`
`130`	`125`
`131`		`- sb.Append(TargetingFilter);`
	`126`	`+ sb.Append(RequestTracingConstants.TargetingFilter);`
`132`	`127`	`}`
`133`	`128`
`134`	`129`	`return sb.ToString();`
`135`	`130`	`}`
`136`	`131`
	`132`	`+ /// <summary>`
	`133`	`+ /// Returns a formatted string containing code names, indicating which tracing features are used by feature flags.`
	`134`	`+ /// </summary>`
	`135`	`+ /// <returns>Formatted string like: "Seed+ConfigRef+Telemetry". If no tracing features are used, empty string will be returned.</returns>`
`137`	`136`	`public string CreateFeaturesString()`
`138`	`137`	`{`
	`138`	`+ if (!UsesAnyTracingFeature())`
	`139`	`+ {`
	`140`	`+ return string.Empty;`
	`141`	`+ }`
	`142`	`+`
`139`	`143`	`var sb = new StringBuilder();`
`140`	`144`
`141`	`145`	`if (UsesSeed)`
Original file line number	Diff line number	Diff line change
`@@ -158,7 +158,7 @@ private static string CreateCorrelationContextHeader(RequestType requestType, Re`
`158`	`158`
`159`	`159`	`if (requestTracingOptions.FeatureFlagTracing.UsesAnyFeatureFilter())`
`160`	`160`	`{`
`161`		`- correlationContextKeyValues.Add(new KeyValuePair<string, string>(RequestTracingConstants.FilterTypeKey, requestTracingOptions.FeatureFlagTracing.CreateFiltersString()));`
	`161`	`+ correlationContextKeyValues.Add(new KeyValuePair<string, string>(RequestTracingConstants.FeatureFlagFilterTypeKey, requestTracingOptions.FeatureFlagTracing.CreateFiltersString()));`
`162`	`162`	`}`
`163`	`163`
`164`	`164`	`if (requestTracingOptions.FeatureFlagTracing.MaxVariants > 0)`
`@@ -181,6 +181,11 @@ private static string CreateCorrelationContextHeader(RequestType requestType, Re`
`181`	`181`	`correlationContextKeyValues.Add(new KeyValuePair<string, string>(RequestTracingConstants.FeatureManagementAspNetCoreVersionKey, requestTracingOptions.FeatureManagementAspNetCoreVersion));`
`182`	`182`	`}`
`183`	`183`
	`184`	`+ if (requestTracingOptions.UsesAnyTracingFeature())`
	`185`	`+ {`
	`186`	`+ correlationContextKeyValues.Add(new KeyValuePair<string, string>(RequestTracingConstants.FeaturesKey, requestTracingOptions.CreateFeaturesString()));`
	`187`	`+ }`
	`188`	`+`
`184`	`189`	`if (requestTracingOptions.IsKeyVaultConfigured)`
`185`	`190`	`{`
`186`	`191`	`correlationContextTags.Add(RequestTracingConstants.KeyVaultConfiguredTag);`
`@@ -191,9 +196,9 @@ private static string CreateCorrelationContextHeader(RequestType requestType, Re`
`191`	`196`	`correlationContextTags.Add(RequestTracingConstants.KeyVaultRefreshConfiguredTag);`
`192`	`197`	`}`
`193`	`198`
`194`		`- if (requestTracingOptions.IsSignalRUsed)`
	`199`	`+ if (requestTracingOptions.IsFailoverRequest)`
`195`	`200`	`{`
`196`		`- correlationContextTags.Add(RequestTracingConstants.SignalRUsedTag);`
	`201`	`+ correlationContextTags.Add(RequestTracingConstants.FailoverRequestTag);`
`197`	`202`	`}`
`198`	`203`
`199`	`204`	`var sb = new StringBuilder();`