From cd996dca90b237ae69e694fb520b66724bc5aaf9 Mon Sep 17 00:00:00 2001 From: Shyam Namboodiripad Date: Fri, 11 Apr 2025 00:52:55 -0700 Subject: [PATCH 1/4] Avoid reading and encoding image data as base64 when he data is already base64 encoded. Also use DataContent.Uri in the above case so that any subsequent calls to evaluate the same image can use the Uri value computed in previous calls instead of recomputing each time. --- .../AIContentExtensions.cs | 31 +++++++++++++++++++ .../ChatMessageExtensions.cs | 16 ++++++++++ .../ChatResponseExtensions.cs | 10 ++++++ .../ContentSafetyServicePayloadUtilities.cs | 31 +++++++------------ 4 files changed, 69 insertions(+), 19 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs new file mode 100644 index 00000000000..06ce5b86ff9 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Extensions.AI.Evaluation.Safety; +internal static class AIContentExtensions +{ + internal static bool IsTextOrUsage(this AIContent content) + => content is TextContent || content is UsageContent; + + internal static bool IsImage(this AIContent content) => + (content is UriContent uriContent && uriContent.HasTopLevelMediaType("image")) || + (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image")); + + internal static bool IsUriBase64Encoded(this DataContent dataContent) + { + ReadOnlyMemory uri = dataContent.Uri.AsMemory(); + + int commaIndex = uri.Span.IndexOf(','); + if (commaIndex == -1) + { + return false; + } + + ReadOnlyMemory metadata = uri.Slice(0, commaIndex); + + bool isBase64Encoded = metadata.Span.EndsWith(";base64".AsSpan(), StringComparison.OrdinalIgnoreCase); + return isBase64Encoded; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs new file mode 100644 index 00000000000..72192adff07 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Extensions.AI.Evaluation.Safety; + +internal static class ChatMessageExtensions +{ + internal static bool ContainsImage(this ChatMessage message) + => message.Contents.Any(c => c.IsImage()); + + internal static bool ContainsImage(this IEnumerable conversation) + => conversation.Any(ContainsImage); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs new file mode 100644 index 00000000000..719fd7310a2 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI.Evaluation.Safety; + +internal static class ChatResponseExtensions +{ + internal static bool ContainsImage(this ChatResponse response) + => response.Messages.ContainsImage(); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs index bb12bc6afec..2ea1204ea48 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs @@ -12,19 +12,6 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; internal static class ContentSafetyServicePayloadUtilities { - internal static bool IsImage(this AIContent content) => - (content is UriContent uriContent && uriContent.HasTopLevelMediaType("image")) || - (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image")); - - internal static bool ContainsImage(this ChatMessage message) - => message.Contents.Any(IsImage); - - internal static bool ContainsImage(this ChatResponse response) - => response.Messages.ContainsImage(); - - internal static bool ContainsImage(this IEnumerable conversation) - => conversation.Any(ContainsImage); - internal static (string payload, IReadOnlyList? diagnostics) GetPayload( ContentSafetyServicePayloadFormat payloadFormat, IEnumerable conversation, @@ -356,8 +343,17 @@ IEnumerable GetContents(ChatMessage message) } else if (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image")) { - BinaryData imageBytes = BinaryData.FromBytes(dataContent.Data); - string base64ImageData = Convert.ToBase64String(imageBytes.ToArray()); + string url; + if (dataContent.IsUriBase64Encoded()) + { + url = dataContent.Uri; + } + else + { + BinaryData imageBytes = BinaryData.FromBytes(dataContent.Data); + string base64ImageData = Convert.ToBase64String(imageBytes.ToArray()); + url = $"data:{dataContent.MediaType};base64,{base64ImageData}"; + } yield return new JsonObject { @@ -365,7 +361,7 @@ IEnumerable GetContents(ChatMessage message) ["image_url"] = new JsonObject { - ["url"] = $"data:{dataContent.MediaType};base64,{base64ImageData}" + ["url"] = url } }; } @@ -582,7 +578,4 @@ void ValidateContents(ChatMessage message) return (turns, normalizedPerTurnContext, diagnostics, contentType); } - - private static bool IsTextOrUsage(this AIContent content) - => content is TextContent || content is UsageContent; } From 18ff9308ba052f9befbe07bb7acd912d5eda8297 Mon Sep 17 00:00:00 2001 From: Shyam Namboodiripad Date: Fri, 11 Apr 2025 01:18:36 -0700 Subject: [PATCH 2/4] Check for supported image types --- .../AIContentExtensions.cs | 17 ++++++++++++++--- .../ChatMessageExtensions.cs | 8 ++++---- .../ChatResponseExtensions.cs | 4 ++-- .../ContentSafetyServicePayloadUtilities.cs | 4 ++-- .../ProtectedMaterialEvaluator.cs | 2 +- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs index 06ce5b86ff9..4e33c64c305 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs @@ -9,9 +9,9 @@ internal static class AIContentExtensions internal static bool IsTextOrUsage(this AIContent content) => content is TextContent || content is UsageContent; - internal static bool IsImage(this AIContent content) => - (content is UriContent uriContent && uriContent.HasTopLevelMediaType("image")) || - (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image")); + internal static bool IsImageWithSupportedFormat(this AIContent content) => + (content is UriContent uriContent && IsSupportedImageFormat(uriContent.MediaType)) || + (content is DataContent dataContent && IsSupportedImageFormat(dataContent.MediaType)); internal static bool IsUriBase64Encoded(this DataContent dataContent) { @@ -28,4 +28,15 @@ internal static bool IsUriBase64Encoded(this DataContent dataContent) bool isBase64Encoded = metadata.Span.EndsWith(";base64".AsSpan(), StringComparison.OrdinalIgnoreCase); return isBase64Encoded; } + + private static bool IsSupportedImageFormat(string mediaType) + { + // 'image/jpeg' is the official MIME type for JPEG. However, some systems recognize 'image/jpg' as well. + + return + mediaType.Equals("image/jpeg", StringComparison.OrdinalIgnoreCase) || + mediaType.Equals("image/jpg", StringComparison.OrdinalIgnoreCase) || + mediaType.Equals("image/png", StringComparison.OrdinalIgnoreCase) || + mediaType.Equals("image/gif", StringComparison.OrdinalIgnoreCase); + } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs index 72192adff07..0634a6bdcba 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs @@ -8,9 +8,9 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; internal static class ChatMessageExtensions { - internal static bool ContainsImage(this ChatMessage message) - => message.Contents.Any(c => c.IsImage()); + internal static bool ContainsImageWithSupportedFormat(this ChatMessage message) + => message.Contents.Any(c => c.IsImageWithSupportedFormat()); - internal static bool ContainsImage(this IEnumerable conversation) - => conversation.Any(ContainsImage); + internal static bool ContainsImageWithSupportedFormat(this IEnumerable conversation) + => conversation.Any(ContainsImageWithSupportedFormat); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs index 719fd7310a2..d8171aaf191 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs @@ -5,6 +5,6 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; internal static class ChatResponseExtensions { - internal static bool ContainsImage(this ChatResponse response) - => response.Messages.ContainsImage(); + internal static bool ContainsImageWithSupportedFormat(this ChatResponse response) + => response.Messages.ContainsImageWithSupportedFormat(); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs index 2ea1204ea48..a2694669106 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs @@ -471,7 +471,7 @@ void ValidateContents(ChatMessage message) if (areImagesSupported) { - if (content.IsImage()) + if (content.IsImageWithSupportedFormat()) { ++imagesCount; } @@ -529,7 +529,7 @@ void ValidateContents(ChatMessage message) EvaluationDiagnostic.Warning( $"The supplied conversation contained {unsupportedContentCount} instances of unsupported content within messages. " + $"The current evaluation being performed by {evaluatorName} only supports content of type '{nameof(TextContent)}', '{nameof(UriContent)}' and '{nameof(DataContent)}'. " + - $"For '{nameof(UriContent)}' and '{nameof(DataContent)}', only content with media type 'image/*' is supported. " + + $"For '{nameof(UriContent)}' and '{nameof(DataContent)}', only content with media type 'image/png', 'image/jpeg' and 'image/gif' are supported. " + $"The unsupported contents were ignored for this evaluation.")); } else diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs index d37bc17c94b..25c99306c32 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs @@ -87,7 +87,7 @@ await EvaluateContentSafetyAsync( // If images are present in the conversation, do a second evaluation for protected material in images. // The content safety service does not support evaluating both text and images in the same request currently. - if (messages.ContainsImage() || modelResponse.ContainsImage()) + if (messages.ContainsImageWithSupportedFormat() || modelResponse.ContainsImageWithSupportedFormat()) { EvaluationResult imageResult = await EvaluateContentSafetyAsync( From a4798ad7233d9d78449eeb30384a32f0665e6320 Mon Sep 17 00:00:00 2001 From: Shyam Namboodiripad Date: Fri, 11 Apr 2025 03:02:42 -0700 Subject: [PATCH 3/4] Update tests to include some images that are supplied via DataContent --- ....Extensions.AI.Evaluation.Integration.Tests.csproj | 11 ++++++++++- .../SafetyEvaluatorTests.cs | 10 +++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj index aff6aadaa2a..4679b04a338 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj @@ -2,9 +2,18 @@ $(LatestTargetFramework) - Microsoft.Extensions.AI.Evaluation.Integration.Tests + Microsoft.Extensions.AI Integration tests for Microsoft.Extensions.AI.Evaluation. + + + + + + + + + diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs index 270c091ecb2..3e4a7867ad5 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs @@ -247,15 +247,15 @@ public async Task EvaluateConversationWithImageInAnswer() await _imageContentSafetyReportingConfiguration.CreateScenarioRunAsync( scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(SafetyEvaluatorTests)}.{nameof(EvaluateConversationWithImageInAnswer)}"); - ChatMessage question = "Can you show me an image pertaining to Microsoft Copilot?".ToUserMessage(); + ChatMessage question = "Can you show me an image pertaining to DotNet?".ToUserMessage(); ChatMessage answer = new ChatMessage { Role = ChatRole.Assistant, Contents = [ - new TextContent("Here's an image pertaining to Microsoft Copilot:"), - new UriContent("https://uhf.microsoft.com/images/banners/RW1iGSh.png", "image/png")], + new TextContent("Here's an image pertaining to DotNet:"), + new DataContent(ImageDataUri.GetImageDataUri())], }; EvaluationResult result = await scenarioRun.EvaluateAsync(question, answer); @@ -280,10 +280,10 @@ await _imageContentSafetyReportingConfiguration.CreateScenarioRunAsync( Role = ChatRole.User, Contents = [ new TextContent("What does this image depict?"), - new UriContent("https://uhf.microsoft.com/images/microsoft/RE1Mu3b.png", "image/png")], + new DataContent(ImageDataUri.GetImageDataUri())], }; - ChatMessage answer1 = "The image depicts a logo for Microsoft Corporation.".ToAssistantMessage(); + ChatMessage answer1 = "The image depicts a logo for DotNet.".ToAssistantMessage(); ChatMessage question2 = "Can you show me an image pertaining to Microsoft Copilot?".ToUserMessage(); From 89861d6297aa653a801ec52df49be8a88243e2e0 Mon Sep 17 00:00:00 2001 From: Shyam Namboodiripad Date: Fri, 11 Apr 2025 03:08:10 -0700 Subject: [PATCH 4/4] Add support to display images in chat conversations --- .../components/ConversationDetails.tsx | 51 ++++++++++++++--- .../TypeScript/components/EvalTypes.d.ts | 13 ++++- .../TypeScript/components/Styles.ts | 4 ++ .../TypeScript/components/Summary.ts | 57 ++++++++++++------- 4 files changed, 96 insertions(+), 29 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ConversationDetails.tsx b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ConversationDetails.tsx index 3e80f03925b..6a3a78d5ca3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ConversationDetails.tsx +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ConversationDetails.tsx @@ -4,8 +4,7 @@ import { useState } from "react"; import ReactMarkdown from "react-markdown"; import { useReportContext } from "./ReportContext"; import { useStyles } from "./Styles"; -import { ChatMessageDisplay } from "./Summary"; - +import { ChatMessageDisplay, isTextContent, isImageContent } from "./Summary"; export const ConversationDetails = ({ messages, model, usage }: { messages: ChatMessageDisplay[]; @@ -25,6 +24,40 @@ export const ConversationDetails = ({ messages, model, usage }: { usage?.totalTokenCount && `Total Tokens: ${usage.totalTokenCount}`, ].filter(Boolean).join(' • '); + const renderContent = (content: AIContent) => { + if (isTextContent(content)) { + return renderMarkdown ? + {content.text} : + 
{content.text}
; + } else if (isImageContent(content)) { + const imageUrl = (content as UriContent).uri || (content as DataContent).uri; + return Content; + } + }; + + const groupMessages = () => { + const result: { role: string, participantName: string, contents: AIContent[] }[] = []; + + for (const message of messages) { + // If this message has the same role and participant as the previous one, append its content + const lastGroup = result[result.length - 1]; + if (lastGroup && lastGroup.role === message.role && lastGroup.participantName === message.participantName) { + lastGroup.contents.push(message.content); + } else { + // Otherwise, start a new group + result.push({ + role: message.role, + participantName: message.participantName, + contents: [message.content] + }); + } + } + + return result; + }; + + const messageGroups = groupMessages(); + return (
setIsExpanded(!isExpanded)}> @@ -35,8 +68,8 @@ export const ConversationDetails = ({ messages, model, usage }: { {isExpanded && (
- {messages.map((message, index) => { - const isFromUserSide = isUserSide(message.role); + {messageGroups.map((group, index) => { + const isFromUserSide = isUserSide(group.role); const messageRowClass = mergeClasses( classes.messageRow, isFromUserSide ? classes.userMessageRow : classes.assistantMessageRow @@ -44,11 +77,13 @@ export const ConversationDetails = ({ messages, model, usage }: { return (
-
{message.participantName}
+
{group.participantName}
- {renderMarkdown ? - {message.content} : - 
{message.content}
} + {group.contents.map((content, contentIndex) => ( +
+ {renderContent(content)} +
+ ))}
); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts index 756d69283d3..c9accb7a90d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts @@ -54,12 +54,23 @@ type AIContent = { $type: string; }; -// TODO: Model other types of AIContent such as function calls, function call results, images, audio etc. +// TODO: Model other types of AIContent such as function calls, function call results, audio etc. type TextContent = AIContent & { $type: "text"; text: string; }; +type UriContent = AIContent & { + $type: "uri"; + uri: string; + mediaType: string; +}; + +type DataContent = AIContent & { + $type: "data"; + uri: string; +}; + type EvaluationResult = { metrics: { [K: string]: MetricWithNoValue | NumericMetric | BooleanMetric | StringMetric; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Styles.ts b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Styles.ts index 7b34280e5dd..2a5cedb7906 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Styles.ts +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Styles.ts @@ -205,6 +205,10 @@ export const useStyles = makeStyles({ preWrap: { whiteSpace: 'pre-wrap', }, + imageContent: { + maxWidth: '100%', + maxHeight: '400px', + }, executionHeaderCell: { display: 'flex', alignItems: 'center', diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts index 16d65ae239b..5b7b57f64e7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts @@ -108,9 +108,15 @@ export class ScoreNode { const { messages } = getConversationDisplay(lastMessage ? [lastMessage] : [], this.scenario?.modelResponse); let history = ""; if (messages.length === 1) { - history = messages[0].content; + const content = messages[0].content; + if (isTextContent(content)) { + history = content.text; + } } else if (messages.length > 1) { - history = messages.map(m => `[${m.participantName}] ${m.content}`).join("\n\n"); + history = messages + .filter(m => isTextContent(m.content)) + .map(m => `[${m.participantName}] ${(m.content as TextContent).text}`) + .join("\n\n"); } this.shortenedPrompt = shortenPrompt(history); @@ -284,10 +290,25 @@ const flattener = function* (node: ScoreNode): Iterable { } }; -const isTextContent = (content: AIContent): content is TextContent => { +export const isTextContent = (content: AIContent): content is TextContent => { return (content as TextContent).text !== undefined; }; +export const isImageContent = (content: AIContent): content is UriContent | DataContent => { + if ((content as UriContent).uri !== undefined && (content as UriContent).mediaType) { + return (content as UriContent).mediaType.startsWith("image/"); + } + + if ((content as DataContent).uri !== undefined) { + const dataContent = content as DataContent; + if (dataContent.uri.startsWith('data:image/')) { + return true; + } + } + + return false; +}; + export type ConversationDisplay = { messages: ChatMessageDisplay[]; model?: string; @@ -297,7 +318,7 @@ export type ConversationDisplay = { export type ChatMessageDisplay = { role: string; participantName: string; - content: string; + content: AIContent; }; export const getConversationDisplay = (messages: ChatMessage[], modelResponse?: ChatResponse): ConversationDisplay => { @@ -305,28 +326,24 @@ export const getConversationDisplay = (messages: ChatMessage[], modelResponse?: for (const m of messages) { for (const c of m.contents) { - if (isTextContent(c)) { - const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role; - chatMessages.push({ - role: m.role, - participantName: participantName, - content: c.text - }); - } + const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role; + chatMessages.push({ + role: m.role, + participantName: participantName, + content: c + }); } } if (modelResponse?.messages) { for (const m of modelResponse.messages) { for (const c of m.contents) { - if (isTextContent(c)) { - const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role || 'Assistant'; - chatMessages.push({ - role: m.role, - participantName: participantName, - content: c.text - }); - } + const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role || 'Assistant'; + chatMessages.push({ + role: m.role, + participantName: participantName, + content: c + }); } } }