Translate OpenAI refusals to ErrorContent (#6393)

Refusals in OpenAI are errors reported when the service can't generate an output that matches the requested schema. Translate refusals to ErrorContent now that we have it.

Author: stephentoub

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs

@@ -155,21 +155,22 @@ void IDisposable.Dispose()
 
                 foreach (var content in input.Contents)
                 {
-                    if (content is FunctionCallContent callRequest)
+                    switch (content)
                     {
-                        message.ToolCalls.Add(
-                            ChatToolCall.CreateFunctionToolCall(
-                                callRequest.CallId,
-                                callRequest.Name,
-                                new(JsonSerializer.SerializeToUtf8Bytes(
-                                    callRequest.Arguments,
-                                    options.GetTypeInfo(typeof(IDictionary<string, object?>))))));
-                    }
-                }
+                        case ErrorContent errorContent when errorContent.ErrorCode is nameof(message.Refusal):
+                            message.Refusal = errorContent.Message;
+                            break;
 
-                if (input.AdditionalProperties?.TryGetValue(nameof(message.Refusal), out string? refusal) is true)
-                {
-                    message.Refusal = refusal;
+                        case FunctionCallContent callRequest:
+                            message.ToolCalls.Add(
+                                ChatToolCall.CreateFunctionToolCall(
+                                    callRequest.CallId,
+                                    callRequest.Name,
+                                    new(JsonSerializer.SerializeToUtf8Bytes(
+                                        callRequest.Arguments,
+                                        options.GetTypeInfo(typeof(IDictionary<string, object?>))))));
+                            break;
+                    }
                 }
 
                 yield return message;
@@ -370,7 +371,7 @@ private static async IAsyncEnumerable<ChatResponseUpdate> FromOpenAIStreamingCha
             // add it to this function calling item.
             if (refusal is not null)
             {
-                (responseUpdate.AdditionalProperties ??= [])[nameof(ChatMessageContentPart.Refusal)] = refusal.ToString();
+                responseUpdate.Contents.Add(new ErrorContent(refusal.ToString()) { ErrorCode = "Refusal" });
             }
 
             // Propagate additional relevant metadata.
@@ -450,6 +451,12 @@ private static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAIComple
             }
         }
 
+        // And add error content for any refusals, which represent errors in generating output that conforms to a provided schema.
+        if (openAICompletion.Refusal is string refusal)
+        {
+            returnMessage.Contents.Add(new ErrorContent(refusal) { ErrorCode = nameof(openAICompletion.Refusal) });
+        }
+
         // Wrap the content in a ChatResponse to return.
         var response = new ChatResponse(returnMessage)
         {
@@ -470,11 +477,6 @@ private static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAIComple
             (response.AdditionalProperties ??= [])[nameof(openAICompletion.ContentTokenLogProbabilities)] = contentTokenLogProbs;
         }
 
-        if (openAICompletion.Refusal is string refusal)
-        {
-            (response.AdditionalProperties ??= [])[nameof(openAICompletion.Refusal)] = refusal;
-        }
-
         if (openAICompletion.RefusalTokenLogProbabilities is { Count: > 0 } refusalTokenLogProbs)
         {
             (response.AdditionalProperties ??= [])[nameof(openAICompletion.RefusalTokenLogProbabilities)] = refusalTokenLogProbs;

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs

@@ -263,6 +263,7 @@ public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
                         MessageId = lastMessageId,
                         ModelId = modelId,
                         ResponseId = responseId,
+                        Role = lastRole,
                         ConversationId = responseId,
                         Contents =
                         [
@@ -274,6 +275,19 @@ public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
                         ],
                     };
                     break;
+
+                case StreamingResponseRefusalDoneUpdate refusalDone:
+                    yield return new ChatResponseUpdate
+                    {
+                        CreatedAt = createdAt,
+                        MessageId = lastMessageId,
+                        ModelId = modelId,
+                        ResponseId = responseId,
+                        Role = lastRole,
+                        ConversationId = responseId,
+                        Contents = [new ErrorContent(refusalDone.Refusal) { ErrorCode = nameof(ResponseContentPart.Refusal) }],
+                    };
+                    break;
             }
         }
     }
@@ -539,9 +553,15 @@ private static List<AIContent> ToAIContents(IEnumerable<ResponseContentPart> con
 
         foreach (ResponseContentPart part in contents)
         {
-            if (part.Kind == ResponseContentPartKind.OutputText)
+            switch (part.Kind)
             {
-                results.Add(new TextContent(part.Text));
+                case ResponseContentPartKind.OutputText:
+                    results.Add(new TextContent(part.Text));
+                    break;
+
+                case ResponseContentPartKind.Refusal:
+                    results.Add(new ErrorContent(part.Refusal) { ErrorCode = nameof(ResponseContentPartKind.Refusal) });
+                    break;
             }
         }
 
@@ -572,6 +592,10 @@ private static List<ResponseContentPart> ToOpenAIResponsesContent(IList<AIConten
                     parts.Add(ResponseContentPart.CreateInputFilePart(null, $"{Guid.NewGuid():N}.pdf",
                         BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(dataContent.Uri, ResponseClientJsonContext.Default.String))));
                     break;
+
+                case ErrorContent errorContent when errorContent.ErrorCode == nameof(ResponseContentPartKind.Refusal):
+                    parts.Add(ResponseContentPart.CreateRefusalPart(errorContent.Message));
+                    break;
             }
         }