Introduce Content Safety evaluators

Author: shyamnamboodiripad

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/MetricCard.tsx

@@ -42,7 +42,7 @@ const useCardStyles = makeStyles({
         padding: '.75rem', 
         border: `1px solid ${tokens.colorNeutralStroke2}`,
         borderRadius: '4px',
-        width: '8rem',
+        width: '12.5rem',
         cursor: 'pointer',
         transition: 'box-shadow 0.2s ease-in-out, outline 0.2s ease-in-out',
         position: 'relative',
@@ -241,4 +241,4 @@ export const MetricDisplay = ({metric}: {metric: MetricWithNoValue | NumericMetr
         classes.metricPill,
     );
     return (<div className={pillClass}><span className={fg}>{metricValue}</span></div>);
-};
+};

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/CodeVulnerabilityEvaluator.cs

@@ -0,0 +1,62 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.Extensions.AI.Evaluation.Safety;
+
+/// <summary>
+/// An <see cref="IEvaluator"/> that utilizes the Azure AI Content Safety service to evaluate code completion responses
+/// produced by an AI model for the presence of vulnerable code.
+/// </summary>
+/// <param name="contentSafetyServiceConfiguration">
+/// Specifies the Azure AI project that should be used and credentials that should be used when this
+/// <see cref="ContentSafetyEvaluator"/> communicates with the Azure AI Content Safety service to perform
+/// evaluations.
+/// </param>
+/// <param name="httpClient">
+/// The <see cref="HttpClient"/> that should be used when communicating with the Azure AI Content Safety service.
+/// While the parameter is optional, it is recommended to supply an <see cref="HttpClient"/> that is configured
+/// with robust resilience and retry policies.
+/// </param>
+public sealed class CodeVulnerabilityEvaluator(
+    ContentSafetyServiceConfiguration contentSafetyServiceConfiguration,
+    HttpClient? httpClient = null)
+        : ContentSafetyEvaluator(
+            evaluatorName: nameof(CodeVulnerabilityEvaluator),
+            metricName: CodeVulnerabilityMetricName,
+            contentSafetyServiceConfiguration,
+            contentSafetyServiceAnnotationTask: "code vulnerability",
+            contentSafetyServiceMetricName: "code_vulnerability",
+            contentSafetyServicePayloadFormat: ContentSafetyServicePayloadFormat.ContextCompletion.ToString(),
+            httpClient)
+{
+    /// <summary>
+    /// Gets the <see cref="EvaluationMetric.Name"/> of the <see cref="BooleanMetric"/> returned by
+    /// <see cref="CodeVulnerabilityEvaluator"/>.
+    /// </summary>
+    public static string CodeVulnerabilityMetricName => "Code Vulnerability";
+
+    /// <inheritdoc/>
+    public override async ValueTask<EvaluationResult> EvaluateAsync(
+        IEnumerable<ChatMessage> messages,
+        ChatResponse modelResponse,
+        ChatConfiguration? chatConfiguration = null,
+        IEnumerable<EvaluationContext>? additionalContext = null,
+        CancellationToken cancellationToken = default)
+    {
+        EvaluationResult result =
+            await EvaluateContentSafetyAsync(
+                messages,
+                modelResponse,
+                cancellationToken: cancellationToken).ConfigureAwait(false);
+
+        result.Interpret(
+            metric => metric is BooleanMetric booleanMetric ? booleanMetric.InterpretScore() : null);
+
+        return result;
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentHarmEvaluator.cs

@@ -0,0 +1,72 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.Extensions.AI.Evaluation.Safety;
+
+#pragma warning disable S1694 // An abstract class should have both abstract and concrete methods
+/// <summary>
+/// An <see langword="abstract"/> base class that can be used to implement <see cref="IEvaluator"/>s that utilize the
+/// Azure AI Content Safety service to evaluate responses produced by an AI model for the presence of a variety of
+/// harmful content such as violence, hate speech, etc.
+/// </summary>
+/// <param name="evaluatorName">The name of the derived <see cref="ContentSafetyEvaluator"/>.</param>
+/// <param name="metricName">
+/// The name of the <see cref="EvaluationMetric"/> produced by the derived <see cref="ContentSafetyEvaluator"/>.
+/// </param>
+/// <param name="contentSafetyServiceConfiguration">
+/// Specifies the Azure AI project that should be used and credentials that should be used when this
+/// <see cref="ContentSafetyEvaluator"/> communicates with the Azure AI Content Safety service to perform evaluations.
+/// </param>
+/// <param name="contentSafetyServiceAnnotationTask">
+/// The name of the annotation task that should be used when this <see cref="ContentSafetyEvaluator"/> communicates
+/// with the Azure AI Content Safety service to perform evaluations.
+/// </param>
+/// <param name="contentSafetyServiceMetricName">
+/// The name of the metric that should be used when this <see cref="ContentSafetyEvaluator"/> communicates with the
+/// Azure AI Content Safety service to perform evaluations.
+/// </param>
+/// <param name="httpClient">
+/// The <see cref="System.Net.Http.HttpClient"/> that should be used when communicating with the Azure AI Content
+/// Safety service. While the parameter is optional, it is recommended to supply an
+/// <see cref="System.Net.Http.HttpClient"/> that is configured with robust resilience and retry policies.
+/// </param>
+public abstract class ContentHarmEvaluator(
+    string evaluatorName,
+    string metricName,
+    ContentSafetyServiceConfiguration contentSafetyServiceConfiguration,
+    string contentSafetyServiceMetricName,
+    HttpClient? httpClient = null)
+        : ContentSafetyEvaluator(
+            evaluatorName,
+            metricName,
+            contentSafetyServiceConfiguration,
+            contentSafetyServiceAnnotationTask: "content harm",
+            contentSafetyServiceMetricName,
+            contentSafetyServicePayloadFormat: ContentSafetyServicePayloadFormat.Conversation.ToString(),
+            httpClient)
+#pragma warning restore S1694
+{
+    /// <inheritdoc/>
+    public sealed override async ValueTask<EvaluationResult> EvaluateAsync(
+        IEnumerable<ChatMessage> messages,
+        ChatResponse modelResponse,
+        ChatConfiguration? chatConfiguration = null,
+        IEnumerable<EvaluationContext>? additionalContext = null,
+        CancellationToken cancellationToken = default)
+    {
+        EvaluationResult result =
+            await EvaluateContentSafetyAsync(
+                messages,
+                modelResponse,
+                cancellationToken: cancellationToken).ConfigureAwait(false);
+
+        result.Interpret(metric => metric is NumericMetric numericMetric ? numericMetric.InterpretHarmScore() : null);
+
+        return result;
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyEvaluator.cs

@@ -0,0 +1,103 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#pragma warning disable S3604
+// S3604: Member initializer values should not be redundant.
+// We disable this warning because it is a false positive arising from the analyzer's lack of support for C#'s primary
+// constructor syntax.
+
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.Extensions.AI.Evaluation.Safety;
+
+/// <summary>
+/// An <see langword="abstract"/> base class that can be used to implement <see cref="IEvaluator"/>s that utilize the
+/// Azure AI Content Safety service to evaluate responses produced by an AI model for the presence of a variety of
+/// unsafe content such as protected material, ungrounded answers, harmful content etc.
+/// </summary>
+/// <param name="evaluatorName">The name of the derived <see cref="ContentSafetyEvaluator"/>.</param>
+/// <param name="metricName">
+/// The name of the <see cref="EvaluationMetric"/> produced by the derived <see cref="ContentSafetyEvaluator"/>.
+/// </param>
+/// <param name="contentSafetyServiceConfiguration">
+/// Specifies the Azure AI project that should be used and credentials that should be used when this
+/// <see cref="ContentSafetyEvaluator"/> communicates with the Azure AI Content Safety service to perform evaluations.
+/// </param>
+/// <param name="contentSafetyServiceAnnotationTask">
+/// The name of the annotation task that should be used when this <see cref="ContentSafetyEvaluator"/> communicates
+/// with the Azure AI Content Safety service to perform evaluations.
+/// </param>
+/// <param name="contentSafetyServiceMetricName">
+/// The name of the metric that should be used when this <see cref="ContentSafetyEvaluator"/> communicates with the
+/// Azure AI Content Safety service to perform evaluations.
+/// </param>
+/// <param name="contentSafetyServicePayloadFormat">
+/// An identifier that specifies the format of the payload that should be used when this
+/// <see cref="ContentSafetyEvaluator"/> communicates with the Azure AI Content Safety service to perform evaluations.
+/// </param>
+/// <param name="httpClient">
+/// The <see cref="System.Net.Http.HttpClient"/> that should be used when communicating with the Azure AI Content
+/// Safety service. While the parameter is optional, it is recommended to supply an
+/// <see cref="System.Net.Http.HttpClient"/> that is configured with robust resilience and retry policies.
+/// </param>
+public abstract class ContentSafetyEvaluator(
+    string evaluatorName,
+    string metricName,
+    ContentSafetyServiceConfiguration contentSafetyServiceConfiguration,
+    string contentSafetyServiceAnnotationTask,
+    string contentSafetyServiceMetricName,
+    string contentSafetyServicePayloadFormat,
+    HttpClient? httpClient = null) : IEvaluator
+{
+    private readonly ContentSafetyService _service =
+        new ContentSafetyService(
+            contentSafetyServiceConfiguration,
+            evaluatorName,
+            contentSafetyServiceAnnotationTask,
+            contentSafetyServiceMetricName,
+#if NET
+            Enum.Parse<ContentSafetyServicePayloadFormat>(contentSafetyServicePayloadFormat),
+#else
+            (ContentSafetyServicePayloadFormat)Enum.Parse(typeof(ContentSafetyServicePayloadFormat), contentSafetyServicePayloadFormat),
+#endif
+            metricName,
+            httpClient);
+
+    /// <inheritdoc/>
+    public IReadOnlyCollection<string> EvaluationMetricNames => [metricName];
+
+    /// <inheritdoc/>
+    public abstract ValueTask<EvaluationResult> EvaluateAsync(
+        IEnumerable<ChatMessage> messages,
+        ChatResponse modelResponse,
+        ChatConfiguration? chatConfiguration = null,
+        IEnumerable<EvaluationContext>? additionalContext = null,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Evaluates the supplied <paramref name="modelResponse"/> using the Azure AI Content Safety Service and returns
+    /// an <see cref="EvaluationResult"/> containing one or more <see cref="EvaluationMetric"/>s.
+    /// </summary>
+    /// <param name="messages">
+    /// The conversation history including the request that produced the supplied <paramref name="modelResponse"/>.
+    /// </param>
+    /// <param name="modelResponse">The response that is to be evaluated.</param>
+    /// <param name="additionalContext">
+    /// Additional contextual information (beyond that which is available in <paramref name="messages"/>) that the
+    /// <see cref="IEvaluator"/> may need to accurately evaluate the supplied <paramref name="modelResponse"/>.
+    /// </param>
+    /// <param name="cancellationToken">
+    /// A <see cref="CancellationToken"/> that can cancel the evaluation operation.
+    /// </param>
+    /// <returns>An <see cref="EvaluationResult"/> containing one or more <see cref="EvaluationMetric"/>s.</returns>
+    protected ValueTask<EvaluationResult> EvaluateContentSafetyAsync(
+        IEnumerable<ChatMessage> messages,
+        ChatResponse modelResponse,
+        IEnumerable<string?>? additionalContext = null,
+        CancellationToken cancellationToken = default)
+            => _service.EvaluateAsync(messages, modelResponse, additionalContext, cancellationToken);
+}