Add TextContent.Annotations

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Text;
 using System.Threading;
@@ -197,14 +198,16 @@ static void Coalesce<TContent>(List<AIContent> contents, Func<string, TContent>
             int start = 0;
             while (start < contents.Count - 1)
             {
-                // We need at least two TextContents in a row to be able to coalesce.
-                if (contents[start] is not TContent firstText)
+                // We need at least two TextContents in a row to be able to coalesce. We also avoid touching contents
+                // that have annotations, as we want to ensure the annotations (and in particular any start/end indices
+                // into the text content) remain accurate.
+                if (!TryAsCoalescable(contents[start], out var firstText))
                 {
                     start++;
                     continue;
                 }
 
-                if (contents[start + 1] is not TContent secondText)
+                if (!TryAsCoalescable(contents[start + 1], out var secondText))
                 {
                     start += 2;
                     continue;
@@ -216,7 +219,7 @@ static void Coalesce<TContent>(List<AIContent> contents, Func<string, TContent>
                 _ = coalescedText.Clear().Append(firstText).Append(secondText);
                 contents[start + 1] = null!;
                 int i = start + 2;
-                for (; i < contents.Count && contents[i] is TContent next; i++)
+                for (; i < contents.Count && TryAsCoalescable(contents[i], out TContent? next); i++)
                 {
                     _ = coalescedText.Append(next);
                     contents[i] = null!;
@@ -230,6 +233,18 @@ static void Coalesce<TContent>(List<AIContent> contents, Func<string, TContent>
                 newContent.AdditionalProperties = firstText.AdditionalProperties?.Clone();
 
                 start = i;
+
+                static bool TryAsCoalescable(AIContent content, [NotNullWhen(true)] out TContent? coalescable)
+                {
+                    if (content is TContent && (content is not TextContent tc || tc.Annotations is not { Count: > 0 }))
+                    {
+                        coalescable = (TContent)content;
+                        return true;
+                    }
+
+                    coalescable = null!;
+                    return false;
+                }
             }
 
             // Remove all of the null slots left over from the coalescing process.

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotation.cs

@@ -0,0 +1,43 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents an annotation on content.
+/// </summary>
+[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
+[JsonDerivedType(typeof(CitationAnnotation), typeDiscriminator: "citation")]
+public class AIAnnotation
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="AIAnnotation"/> class.
+    /// </summary>
+    public AIAnnotation()
+    {
+    }
+
+    /// <summary>Gets or sets any target regions for the annotation, pointing to where in the associated <see cref="AIContent"/> this annotation applies.</summary>
+    /// <remarks>
+    /// The most common form of <see cref="AnnotatedRegion"/> is <see cref="TextSpanAnnotatedRegion"/>, which provides starting and ending character indices
+    /// for <see cref="TextContent"/>.
+    /// </remarks>
+    public IList<AnnotatedRegion>? AnnotatedRegions { get; set; }
+
+    /// <summary>Gets or sets the raw representation of the annotation from an underlying implementation.</summary>
+    /// <remarks>
+    /// If an <see cref="AIAnnotation"/> is created to represent some underlying object from another object
+    /// model, this property can be used to store that original object. This can be useful for debugging or
+    /// for enabling a consumer to access the underlying object model, if needed.
+    /// </remarks>
+    [JsonIgnore]
+    public object? RawRepresentation { get; set; }
+
+    /// <summary>
+    /// Gets or sets additional metadata specific to the provider or source type.
+    /// </summary>
+    public AdditionalPropertiesDictionary? AdditionalProperties { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections.Generic;
 using System.Text.Json.Serialization;
 
 namespace Microsoft.Extensions.AI;
@@ -24,6 +25,11 @@ public AIContent()
     {
     }
 
+    /// <summary>
+    /// Gets or sets a list of annotations on this content.
+    /// </summary>
+    public IList<AIAnnotation>? Annotations { get; set; }
+
     /// <summary>Gets or sets the raw representation of the content from an underlying implementation.</summary>
     /// <remarks>
     /// If an <see cref="AIContent"/> is created to represent some underlying object from another object

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AnnotatedRegion.cs

@@ -0,0 +1,23 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Text.Json.Serialization;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>Describes the portion of an associated <see cref="AIContent"/> to which an annotation applies.</summary>
+/// <remarks>
+/// Details about the region is provided by derived types based on how the region is described. For example, starting
+/// and ending indices into text content are provided by <see cref="TextSpanAnnotatedRegion"/>.
+/// </remarks>
+[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
+[JsonDerivedType(typeof(TextSpanAnnotatedRegion), typeDiscriminator: "textSpan")]
+public class AnnotatedRegion
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="AnnotatedRegion"/> class.
+    /// </summary>
+    public AnnotatedRegion()
+    {
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CitationAnnotation.cs

@@ -0,0 +1,53 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents an annotation that links content to source references,
+/// such as documents, URLs, files, or tool outputs.
+/// </summary>
+public class CitationAnnotation : AIAnnotation
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="CitationAnnotation"/> class.
+    /// </summary>
+    public CitationAnnotation()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the title or name of the source.
+    /// </summary>
+    /// <remarks>
+    /// This could be the title of a document, a title from a web page, a name of a file, or similarly descriptive text.
+    /// </remarks>
+    public string? Title { get; set; }
+
+    /// <summary>
+    /// Gets or sets a URI from which the source material was retrieved.
+    /// </summary>
+    public Uri? Url { get; set; }
+
+    /// <summary>Gets or sets a source identifier associated with the annotation.</summary>
+    /// <remarks>
+    /// This is a provider-specific identifier that can be used to reference the source material by
+    /// an ID. This may be a document ID, or a file ID, or some other identifier for the source material
+    /// that can be used to uniquely identify it with the provider.
+    /// </remarks>
+    public string? FileId { get; set; }
+
+    /// <summary>Gets or sets the name of any tool involved in the production of the associated content.</summary>
+    /// <remarks>
+    /// This might be a function name, such as one from <see cref="AITool.Name"/>, or the name of a built-in tool
+    /// from the provider, such as "code_interpreter" or "file_search".
+    /// </remarks>
+    public string? ToolName { get; set; }
+
+    /// <summary>
+    /// Gets or sets a snippet or excerpt from the source that was cited.
+    /// </summary>
+    public string? Snippet { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/TextSpanAnnotatedRegion.cs

@@ -0,0 +1,32 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>Describes a location in the associated <see cref="AIContent"/> based on starting and ending character indices.</summary>
+/// <remarks>This <see cref="AnnotatedRegion"/> typically applies to <see cref="TextContent"/>.</remarks>
+[DebuggerDisplay("[{StartIndex}, {EndIndex})")]
+public sealed class TextSpanAnnotatedRegion : AnnotatedRegion
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TextSpanAnnotatedRegion"/> class.
+    /// </summary>
+    public TextSpanAnnotatedRegion()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the start character index (inclusive) of the annotated span in the <see cref="AIContent"/>.
+    /// </summary>
+    [JsonPropertyName("start")]
+    public int? StartIndex { get; set; }
+
+    /// <summary>
+    /// Gets or sets the end character index (exclusive) of the annotated span in the <see cref="AIContent"/>.
+    /// </summary>
+    [JsonPropertyName("end")]
+    public int? EndIndex { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json

@@ -123,6 +123,30 @@
         }
       ]
     },
+    {
+      "Type": "class Microsoft.Extensions.AI.AIAnnotation",
+      "Stage": "Stable",
+      "Methods": [
+        {
+          "Member": "Microsoft.Extensions.AI.AIAnnotation.AIAnnotation();",
+          "Stage": "Stable"
+        }
+      ],
+      "Properties": [
+        {
+          "Member": "System.Collections.Generic.IList<Microsoft.Extensions.AI.AnnotatedRegion>? Microsoft.Extensions.AI.AIAnnotation.AnnotatedRegions { get; set; }",
+          "Stage": "Stable"
+        },
+        {
+          "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.AIAnnotation.AdditionalProperties { get; set; }",
+          "Stage": "Stable"
+        },
+        {
+          "Member": "object? Microsoft.Extensions.AI.AIAnnotation.RawRepresentation { get; set; }",
+          "Stage": "Stable"
+        }
+      ]
+    },
     {
       "Type": "class Microsoft.Extensions.AI.AIContent",
       "Stage": "Stable",
@@ -137,6 +161,10 @@
           "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.AIContent.AdditionalProperties { get; set; }",
           "Stage": "Stable"
         },
+        {
+          "Member": "System.Collections.Generic.IList<Microsoft.Extensions.AI.AIAnnotation>? Microsoft.Extensions.AI.AIContent.Annotations { get; set; }",
+          "Stage": "Stable"
+        },
         {
           "Member": "object? Microsoft.Extensions.AI.AIContent.RawRepresentation { get; set; }",
           "Stage": "Stable"
@@ -653,6 +681,16 @@
         }
       ]
     },
+    {
+      "Type": "class Microsoft.Extensions.AI.AnnotatedRegion",
+      "Stage": "Stable",
+      "Methods": [
+        {
+          "Member": "Microsoft.Extensions.AI.AnnotatedRegion.AnnotatedRegion();",
+          "Stage": "Stable"
+        }
+      ]
+    },
     {
       "Type": "sealed class Microsoft.Extensions.AI.AutoChatToolMode : Microsoft.Extensions.AI.ChatToolMode",
       "Stage": "Stable",
@@ -1323,6 +1361,38 @@
         }
       ]
     },
+    {
+      "Type": "class Microsoft.Extensions.AI.CitationAnnotation : Microsoft.Extensions.AI.AIAnnotation",
+      "Stage": "Stable",
+      "Methods": [
+        {
+          "Member": "Microsoft.Extensions.AI.CitationAnnotation.CitationAnnotation();",
+          "Stage": "Stable"
+        }
+      ],
+      "Properties": [
+        {
+          "Member": "string? Microsoft.Extensions.AI.CitationAnnotation.Title { get; set; }",
+          "Stage": "Stable"
+        },
+        {
+          "Member": "string? Microsoft.Extensions.AI.CitationAnnotation.ToolName { get; set; }",
+          "Stage": "Stable"
+        },
+        {
+          "Member": "System.Uri? Microsoft.Extensions.AI.CitationAnnotation.Url { get; set; }",
+          "Stage": "Stable"
+        },
+        {
+          "Member": "string? Microsoft.Extensions.AI.CitationAnnotation.FileId { get; set; }",
+          "Stage": "Stable"
+        },
+        {
+          "Member": "string? Microsoft.Extensions.AI.CitationAnnotation.Snippet { get; set; }",
+          "Stage": "Stable"
+        }
+      ]
+    },
     {
       "Type": "class Microsoft.Extensions.AI.DataContent : Microsoft.Extensions.AI.AIContent",
       "Stage": "Stable",
@@ -2273,6 +2343,26 @@
         }
       ]
     },
+    {
+      "Type": "sealed class Microsoft.Extensions.AI.TextSpanAnnotatedRegion : Microsoft.Extensions.AI.AnnotatedRegion",
+      "Stage": "Stable",
+      "Methods": [
+        {
+          "Member": "Microsoft.Extensions.AI.TextSpanAnnotatedRegion.TextSpanAnnotatedRegion();",
+          "Stage": "Stable"
+        }
+      ],
+      "Properties": [
+        {
+          "Member": "int? Microsoft.Extensions.AI.TextSpanAnnotatedRegion.StartIndex { get; set; }",
+          "Stage": "Stable"
+        },
+        {
+          "Member": "int? Microsoft.Extensions.AI.TextSpanAnnotatedRegion.EndIndex { get; set; }",
+          "Stage": "Stable"
+        }
+      ]
+    },
     {
       "Type": "class Microsoft.Extensions.AI.UriContent : Microsoft.Extensions.AI.AIContent",
       "Stage": "Stable",