Display images in conversation for report

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ConversationDetails.tsx

@@ -4,8 +4,7 @@ import { useState } from "react";
 import ReactMarkdown from "react-markdown";
 import { useReportContext } from "./ReportContext";
 import { useStyles } from "./Styles";
-import { ChatMessageDisplay } from "./Summary";
-
+import { ChatMessageDisplay, isTextContent, isImageContent } from "./Summary";
 
 export const ConversationDetails = ({ messages, model, usage }: {
     messages: ChatMessageDisplay[];
@@ -25,6 +24,40 @@ export const ConversationDetails = ({ messages, model, usage }: {
         usage?.totalTokenCount && `Total Tokens: ${usage.totalTokenCount}`,
     ].filter(Boolean).join(' • ');
 
+    const renderContent = (content: AIContent) => {
+        if (isTextContent(content)) {
+            return renderMarkdown ?
+                <ReactMarkdown>{content.text}</ReactMarkdown> :
+                <pre className={classes.preWrap}>{content.text}</pre>;
+        } else if (isImageContent(content)) {
+            const imageUrl = (content as UriContent).uri || (content as DataContent).uri;
+            return <img src={imageUrl} alt="Content" className={classes.imageContent} />;
+        }
+    };
+
+    const groupMessages = () => {
+        const result: { role: string, participantName: string, contents: AIContent[] }[] = [];
+
+        for (const message of messages) {
+            // If this message has the same role and participant as the previous one, append its content
+            const lastGroup = result[result.length - 1];
+            if (lastGroup && lastGroup.role === message.role && lastGroup.participantName === message.participantName) {
+                lastGroup.contents.push(message.content);
+            } else {
+                // Otherwise, start a new group
+                result.push({
+                    role: message.role,
+                    participantName: message.participantName,
+                    contents: [message.content]
+                });
+            }
+        }
+
+        return result;
+    };
+
+    const messageGroups = groupMessages();
+
     return (
         <div className={classes.section}>
             <div className={classes.sectionHeader} onClick={() => setIsExpanded(!isExpanded)}>
@@ -35,20 +68,22 @@ export const ConversationDetails = ({ messages, model, usage }: {
 
             {isExpanded && (
                 <div className={classes.sectionContainer}>
-                    {messages.map((message, index) => {
-                        const isFromUserSide = isUserSide(message.role);
+                    {messageGroups.map((group, index) => {
+                        const isFromUserSide = isUserSide(group.role);
                         const messageRowClass = mergeClasses(
                             classes.messageRow,
                             isFromUserSide ? classes.userMessageRow : classes.assistantMessageRow
                         );
 
                         return (
                             <div key={index} className={messageRowClass}>
-                                <div className={classes.messageParticipantName}>{message.participantName}</div>
+                                <div className={classes.messageParticipantName}>{group.participantName}</div>
                                 <div className={classes.messageBubble}>
-                                    {renderMarkdown ?
-                                        <ReactMarkdown>{message.content}</ReactMarkdown> :
-                                        <pre className={classes.preWrap}>{message.content}</pre>}
+                                    {group.contents.map((content, contentIndex) => (
+                                        <div key={contentIndex}>
+                                            {renderContent(content)}
+                                        </div>
+                                    ))}
                                 </div>
                             </div>
                         );

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts

@@ -54,12 +54,23 @@ type AIContent = {
     $type: string;
 };
 
-// TODO: Model other types of AIContent such as function calls, function call results, images, audio etc.
+// TODO: Model other types of AIContent such as function calls, function call results, audio etc.
 type TextContent = AIContent & {
     $type: "text";
     text: string;
 };
 
+type UriContent = AIContent & {
+    $type: "uri";
+    uri: string;
+    mediaType: string;
+};
+
+type DataContent = AIContent & {
+    $type: "data";
+    uri: string;
+};
+
 type EvaluationResult = {
     metrics: {
         [K: string]: MetricWithNoValue | NumericMetric | BooleanMetric | StringMetric;

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Styles.ts

@@ -205,6 +205,10 @@ export const useStyles = makeStyles({
     preWrap: {
         whiteSpace: 'pre-wrap',
     },
+    imageContent: {
+        maxWidth: '100%',
+        maxHeight: '400px',
+    },
     executionHeaderCell: {
         display: 'flex',
         alignItems: 'center',

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts

@@ -108,9 +108,15 @@ export class ScoreNode {
             const { messages } = getConversationDisplay(lastMessage ? [lastMessage] : [], this.scenario?.modelResponse);
             let history = "";
             if (messages.length === 1) {
-                history = messages[0].content;
+                const content = messages[0].content;
+                if (isTextContent(content)) {
+                    history = content.text;
+                }
             } else if (messages.length > 1) {
-                history = messages.map(m => `[${m.participantName}] ${m.content}`).join("\n\n");
+                history = messages
+                    .filter(m => isTextContent(m.content))
+                    .map(m => `[${m.participantName}] ${(m.content as TextContent).text}`)
+                    .join("\n\n");
             }
 
             this.shortenedPrompt = shortenPrompt(history);
@@ -284,10 +290,25 @@ const flattener = function* (node: ScoreNode): Iterable<ScoreNode> {
     }
 };
 
-const isTextContent = (content: AIContent): content is TextContent => {
+export const isTextContent = (content: AIContent): content is TextContent => {
     return (content as TextContent).text !== undefined;
 };
 
+export const isImageContent = (content: AIContent): content is UriContent | DataContent => {
+    if ((content as UriContent).uri !== undefined && (content as UriContent).mediaType) {
+        return (content as UriContent).mediaType.startsWith("image/");
+    }
+
+    if ((content as DataContent).uri !== undefined) {
+        const dataContent = content as DataContent;
+        if (dataContent.uri.startsWith('data:image/')) {
+            return true;
+        }
+    }
+
+    return false;
+};
+
 export type ConversationDisplay = {
     messages: ChatMessageDisplay[];
     model?: string;
@@ -297,36 +318,32 @@ export type ConversationDisplay = {
 export type ChatMessageDisplay = {
     role: string;
     participantName: string;
-    content: string;
+    content: AIContent;
 };
 
 export const getConversationDisplay = (messages: ChatMessage[], modelResponse?: ChatResponse): ConversationDisplay => {
     const chatMessages: ChatMessageDisplay[] = [];
 
     for (const m of messages) {
         for (const c of m.contents) {
-            if (isTextContent(c)) {
-                const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role;
-                chatMessages.push({
-                    role: m.role,
-                    participantName: participantName,
-                    content: c.text
-                });
-            }
+            const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role;
+            chatMessages.push({
+                role: m.role,
+                participantName: participantName,
+                content: c
+            });
         }
     }
 
     if (modelResponse?.messages) {
         for (const m of modelResponse.messages) {
             for (const c of m.contents) {
-                if (isTextContent(c)) {
-                    const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role || 'Assistant';
-                    chatMessages.push({
-                        role: m.role,
-                        participantName: participantName,
-                        content: c.text
-                    });
-                }
+                const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role || 'Assistant';
+                chatMessages.push({
+                    role: m.role,
+                    participantName: participantName,
+                    content: c
+                });
             }
         }
     }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs

@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.Extensions.AI.Evaluation.Safety;
+internal static class AIContentExtensions
+{
+    internal static bool IsTextOrUsage(this AIContent content)
+        => content is TextContent || content is UsageContent;
+
+    internal static bool IsImageWithSupportedFormat(this AIContent content) =>
+        (content is UriContent uriContent && IsSupportedImageFormat(uriContent.MediaType)) ||
+        (content is DataContent dataContent && IsSupportedImageFormat(dataContent.MediaType));
+
+    internal static bool IsUriBase64Encoded(this DataContent dataContent)
+    {
+        ReadOnlyMemory<char> uri = dataContent.Uri.AsMemory();
+
+        int commaIndex = uri.Span.IndexOf(',');
+        if (commaIndex == -1)
+        {
+            return false;
+        }
+
+        ReadOnlyMemory<char> metadata = uri.Slice(0, commaIndex);
+
+        bool isBase64Encoded = metadata.Span.EndsWith(";base64".AsSpan(), StringComparison.OrdinalIgnoreCase);
+        return isBase64Encoded;
+    }
+
+    private static bool IsSupportedImageFormat(string mediaType)
+    {
+        // 'image/jpeg' is the official MIME type for JPEG. However, some systems recognize 'image/jpg' as well.
+
+        return
+            mediaType.Equals("image/jpeg", StringComparison.OrdinalIgnoreCase) ||
+            mediaType.Equals("image/jpg", StringComparison.OrdinalIgnoreCase) ||
+            mediaType.Equals("image/png", StringComparison.OrdinalIgnoreCase) ||
+            mediaType.Equals("image/gif", StringComparison.OrdinalIgnoreCase);
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs

@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.Extensions.AI.Evaluation.Safety;
+
+internal static class ChatMessageExtensions
+{
+    internal static bool ContainsImageWithSupportedFormat(this ChatMessage message)
+        => message.Contents.Any(c => c.IsImageWithSupportedFormat());
+
+    internal static bool ContainsImageWithSupportedFormat(this IEnumerable<ChatMessage> conversation)
+        => conversation.Any(ContainsImageWithSupportedFormat);
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs

@@ -0,0 +1,10 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.Extensions.AI.Evaluation.Safety;
+
+internal static class ChatResponseExtensions
+{
+    internal static bool ContainsImageWithSupportedFormat(this ChatResponse response)
+        => response.Messages.ContainsImageWithSupportedFormat();
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs

@@ -12,19 +12,6 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety;
 
 internal static class ContentSafetyServicePayloadUtilities
 {
-    internal static bool IsImage(this AIContent content) =>
-        (content is UriContent uriContent && uriContent.HasTopLevelMediaType("image")) ||
-        (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image"));
-
-    internal static bool ContainsImage(this ChatMessage message)
-        => message.Contents.Any(IsImage);
-
-    internal static bool ContainsImage(this ChatResponse response)
-        => response.Messages.ContainsImage();
-
-    internal static bool ContainsImage(this IEnumerable<ChatMessage> conversation)
-        => conversation.Any(ContainsImage);
-
     internal static (string payload, IReadOnlyList<EvaluationDiagnostic>? diagnostics) GetPayload(
         ContentSafetyServicePayloadFormat payloadFormat,
         IEnumerable<ChatMessage> conversation,
@@ -356,16 +343,25 @@ IEnumerable<JsonObject> GetContents(ChatMessage message)
                     }
                     else if (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image"))
                     {
-                        BinaryData imageBytes = BinaryData.FromBytes(dataContent.Data);
-                        string base64ImageData = Convert.ToBase64String(imageBytes.ToArray());
+                        string url;
+                        if (dataContent.IsUriBase64Encoded())
+                        {
+                            url = dataContent.Uri;
+                        }
+                        else
+                        {
+                            BinaryData imageBytes = BinaryData.FromBytes(dataContent.Data);
+                            string base64ImageData = Convert.ToBase64String(imageBytes.ToArray());
+                            url = $"data:{dataContent.MediaType};base64,{base64ImageData}";
+                        }
 
                         yield return new JsonObject
                         {
                             ["type"] = "image_url",
                             ["image_url"] =
                                 new JsonObject
                                 {
-                                    ["url"] = $"data:{dataContent.MediaType};base64,{base64ImageData}"
+                                    ["url"] = url
                                 }
                         };
                     }
@@ -475,7 +471,7 @@ void ValidateContents(ChatMessage message)
 
                 if (areImagesSupported)
                 {
-                    if (content.IsImage())
+                    if (content.IsImageWithSupportedFormat())
                     {
                         ++imagesCount;
                     }
@@ -533,7 +529,7 @@ void ValidateContents(ChatMessage message)
                     EvaluationDiagnostic.Warning(
                         $"The supplied conversation contained {unsupportedContentCount} instances of unsupported content within messages. " +
                         $"The current evaluation being performed by {evaluatorName} only supports content of type '{nameof(TextContent)}', '{nameof(UriContent)}' and '{nameof(DataContent)}'. " +
-                        $"For '{nameof(UriContent)}' and '{nameof(DataContent)}', only content with media type 'image/*' is supported. " +
+                        $"For '{nameof(UriContent)}' and '{nameof(DataContent)}', only content with media type 'image/png', 'image/jpeg' and 'image/gif' are supported. " +
                         $"The unsupported contents were ignored for this evaluation."));
             }
             else
@@ -582,7 +578,4 @@ void ValidateContents(ChatMessage message)
 
         return (turns, normalizedPerTurnContext, diagnostics, contentType);
     }
-
-    private static bool IsTextOrUsage(this AIContent content)
-        => content is TextContent || content is UsageContent;
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs

@@ -87,7 +87,7 @@ await EvaluateContentSafetyAsync(
 
         // If images are present in the conversation, do a second evaluation for protected material in images.
         // The content safety service does not support evaluating both text and images in the same request currently.
-        if (messages.ContainsImage() || modelResponse.ContainsImage())
+        if (messages.ContainsImageWithSupportedFormat() || modelResponse.ContainsImageWithSupportedFormat())
         {
             EvaluationResult imageResult =
                 await EvaluateContentSafetyAsync(

test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj

@@ -2,9 +2,18 @@
 
   <PropertyGroup>
     <TargetFrameworks>$(LatestTargetFramework)</TargetFrameworks>
-    <RootNamespace>Microsoft.Extensions.AI.Evaluation.Integration.Tests</RootNamespace>
+    <RootNamespace>Microsoft.Extensions.AI</RootNamespace>
     <Description>Integration tests for Microsoft.Extensions.AI.Evaluation.</Description>
   </PropertyGroup>
+
+  <ItemGroup>
+    <EmbeddedResource Include="..\..\Shared\ImageDataUri\dotnet.png" Link="Resources\dotnet.png"/>
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="..\..\Shared\ImageDataUri\ImageDataUri.cs" Link="Shared\ImageDataUri\ImageDataUri.cs" />
+  </ItemGroup>
+
 
   <ItemGroup>
     <PackageReference Include="Azure.AI.OpenAI" />