Skip to content

Commit bb3c065

Browse files
crickmanrogerbarreto
crickman
and
rogerbarreto
authored
.Net Agents - Add support for URL citation on Azure Agent (#11910)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> Expose URL citation for `AzureAIAgent` Fixes: #11635 ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> Tools such as the `bing-grounding-tool` provide url based annotations. This change updates the SDK, run processing, and annotation content types. - SDK: Must reference a version of that includes the fix for streaming url annocations - Run Processing: Capture annotation data and expose via the SK content model - Content Types: Add support for url annotation to `AnnotationContent` and `StreamingAnnocationContent` ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [X] The code builds clean without any errors or warnings - [X] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [X] All unit tests pass, and I have added new tests where possible - [X] I didn't break anyone 😄 --------- Co-authored-by: Roger Barreto <[email protected]>
1 parent 37800ff commit bb3c065

20 files changed

+778
-194
lines changed

‎dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step09_AzureAIAgent_BingGrounding.cs‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
using Azure.AI.Projects;
3+
using Microsoft.SemanticKernel;
4+
using Microsoft.SemanticKernel.Agents;
5+
using Microsoft.SemanticKernel.Agents.AzureAI;
6+
using Microsoft.SemanticKernel.ChatCompletion;
7+
using FoundryAgent = Azure.AI.Projects.Agent;
8+
9+
namespace GettingStarted.AzureAgents;
10+
11+
/// <summary>
12+
/// Demonstrate using code-interpreter on <see cref="AzureAIAgent"/> .
13+
/// </summary>
14+
public class Step09_AzureAIAgent_BingGrounding(ITestOutputHelper output) : BaseAzureAgentTest(output)
15+
{
16+
[Fact]
17+
public async Task UseBingGroundingToolWithAgent()
18+
{
19+
// Access the BingGrounding connection
20+
ConnectionsClient connectionsClient = this.Client.GetConnectionsClient();
21+
ConnectionResponse bingConnection = await connectionsClient.GetConnectionAsync(TestConfiguration.AzureAI.BingConnectionId);
22+
23+
// Define the agent
24+
ToolConnectionList toolConnections = new()
25+
{
26+
ConnectionList = { new ToolConnection(bingConnection.Id) }
27+
};
28+
FoundryAgent definition = await this.AgentsClient.CreateAgentAsync(
29+
TestConfiguration.AzureAI.ChatModelId,
30+
tools: [new BingGroundingToolDefinition(toolConnections)]);
31+
AzureAIAgent agent = new(definition, this.AgentsClient);
32+
33+
// Create a thread for the agent conversation.
34+
AzureAIAgentThread thread = new(this.AgentsClient, metadata: SampleMetadata);
35+
36+
// Respond to user input
37+
try
38+
{
39+
//await InvokeAgentAsync("How does wikipedia explain Euler's Identity?");
40+
await InvokeAgentAsync("What is the current price of gold?");
41+
}
42+
finally
43+
{
44+
await thread.DeleteAsync();
45+
await this.AgentsClient.DeleteAgentAsync(agent.Id);
46+
}
47+
48+
// Local function to invoke agent and display the conversation messages.
49+
async Task InvokeAgentAsync(string input)
50+
{
51+
ChatMessageContent message = new(AuthorRole.User, input);
52+
this.WriteAgentChatMessage(message);
53+
54+
await foreach (ChatMessageContent response in agent.InvokeAsync(message, thread))
55+
{
56+
this.WriteAgentChatMessage(response);
57+
}
58+
}
59+
}
60+
61+
[Fact]
62+
public async Task UseBingGroundingToolWithStreaming()
63+
{
64+
// Access the BingGrounding connection
65+
ConnectionsClient connectionClient = this.Client.GetConnectionsClient();
66+
ConnectionResponse bingConnectionResponse = await connectionClient.GetConnectionAsync(TestConfiguration.AzureAI.BingConnectionId);
67+
68+
// Define the agent
69+
ToolConnectionList toolConnections = new()
70+
{
71+
ConnectionList = { new ToolConnection(bingConnectionResponse.Id) }
72+
};
73+
FoundryAgent definition = await this.AgentsClient.CreateAgentAsync(
74+
TestConfiguration.AzureAI.ChatModelId,
75+
tools: [new BingGroundingToolDefinition(toolConnections)]);
76+
AzureAIAgent agent = new(definition, this.AgentsClient);
77+
78+
// Create a thread for the agent conversation.
79+
AzureAIAgentThread thread = new(this.AgentsClient, metadata: SampleMetadata);
80+
81+
// Respond to user input
82+
try
83+
{
84+
await InvokeAgentAsync("What is the current price of gold?");
85+
86+
// Display chat history
87+
Console.WriteLine("\n================================");
88+
Console.WriteLine("CHAT HISTORY");
89+
Console.WriteLine("================================");
90+
91+
await foreach (ChatMessageContent message in thread.GetMessagesAsync())
92+
{
93+
this.WriteAgentChatMessage(message);
94+
}
95+
}
96+
finally
97+
{
98+
await thread.DeleteAsync();
99+
await this.AgentsClient.DeleteAgentAsync(agent.Id);
100+
}
101+
102+
// Local function to invoke agent and display the conversation messages.
103+
async Task InvokeAgentAsync(string input)
104+
{
105+
ChatMessageContent message = new(AuthorRole.User, input);
106+
this.WriteAgentChatMessage(message);
107+
108+
bool isFirst = false;
109+
await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(message, thread))
110+
{
111+
if (!isFirst)
112+
{
113+
Console.WriteLine($"\n# {response.Role} - {response.AuthorName ?? "*"}:");
114+
isFirst = true;
115+
}
116+
117+
if (!string.IsNullOrWhiteSpace(response.Content))
118+
{
119+
Console.WriteLine($"\t> streamed: {response.Content}");
120+
}
121+
122+
foreach (StreamingAnnotationContent? annotation in response.Items.OfType<StreamingAnnotationContent>())
123+
{
124+
Console.WriteLine($"\t {annotation.ReferenceId} - {annotation.Title}");
125+
}
126+
}
127+
}
128+
}
129+
}

‎dotnet/src/Agents/AzureAI/Internal/AgentThreadActions.cs‎

Lines changed: 81 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ await functionProcessor.InvokeFunctionCallsAsync(
274274

275275
if (message is not null)
276276
{
277-
ChatMessageContent content = GenerateMessageContent(agent.GetName(), message, completedStep);
277+
ChatMessageContent content = GenerateMessageContent(agent.GetName(), message, completedStep, logger);
278278

279279
if (content.Items.Count > 0)
280280
{
@@ -415,7 +415,7 @@ public static async IAsyncEnumerable<StreamingChatMessageContent> InvokeStreamin
415415
switch (contentUpdate.UpdateKind)
416416
{
417417
case StreamingUpdateReason.MessageUpdated:
418-
yield return GenerateStreamingMessageContent(agent.GetName(), contentUpdate);
418+
yield return GenerateStreamingMessageContent(agent.GetName(), run!, contentUpdate, logger);
419419
break;
420420
}
421421
}
@@ -524,7 +524,7 @@ await RetrieveMessageAsync(
524524

525525
if (message != null)
526526
{
527-
ChatMessageContent content = GenerateMessageContent(agent.GetName(), message, step);
527+
ChatMessageContent content = GenerateMessageContent(agent.GetName(), message, step, logger);
528528
messages?.Add(content);
529529
}
530530
}
@@ -555,7 +555,7 @@ await RetrieveMessageAsync(
555555
logger.LogAzureAIAgentCompletedRun(nameof(InvokeAsync), run?.Id ?? "Failed", threadId);
556556
}
557557

558-
private static ChatMessageContent GenerateMessageContent(string? assistantName, ThreadMessage message, RunStep? completedStep = null)
558+
private static ChatMessageContent GenerateMessageContent(string? assistantName, ThreadMessage message, RunStep? completedStep = null, ILogger? logger = null)
559559
{
560560
AuthorRole role = new(message.Role.ToString());
561561

@@ -591,7 +591,15 @@ private static ChatMessageContent GenerateMessageContent(string? assistantName,
591591

592592
foreach (MessageTextAnnotation annotation in textContent.Annotations)
593593
{
594-
content.Items.Add(GenerateAnnotationContent(annotation));
594+
AnnotationContent? annotationItem = GenerateAnnotationContent(annotation);
595+
if (annotationItem != null)
596+
{
597+
content.Items.Add(annotationItem);
598+
}
599+
else
600+
{
601+
logger?.LogAzureAIAgentUnknownAnnotation(nameof(GenerateMessageContent), message.RunId, message.ThreadId, annotation.GetType());
602+
}
595603
}
596604
}
597605
// Process image content
@@ -604,7 +612,7 @@ private static ChatMessageContent GenerateMessageContent(string? assistantName,
604612
return content;
605613
}
606614

607-
private static StreamingChatMessageContent GenerateStreamingMessageContent(string? assistantName, MessageContentUpdate update)
615+
private static StreamingChatMessageContent GenerateStreamingMessageContent(string? assistantName, ThreadRun run, MessageContentUpdate update, ILogger? logger)
608616
{
609617
StreamingChatMessageContent content =
610618
new(AuthorRole.Assistant, content: null)
@@ -625,7 +633,15 @@ private static StreamingChatMessageContent GenerateStreamingMessageContent(strin
625633
// Process annotations
626634
else if (update.TextAnnotation != null)
627635
{
628-
content.Items.Add(GenerateStreamingAnnotationContent(update.TextAnnotation));
636+
StreamingAnnotationContent? annotationItem = GenerateStreamingAnnotationContent(update.TextAnnotation);
637+
if (annotationItem != null)
638+
{
639+
content.Items.Add(annotationItem);
640+
}
641+
else
642+
{
643+
logger?.LogAzureAIAgentUnknownAnnotation(nameof(GenerateStreamingMessageContent), run.Id, run.ThreadId, update.TextAnnotation.GetType());
644+
}
629645
}
630646

631647
if (update.Role.HasValue && update.Role.Value != MessageRole.User)
@@ -665,46 +681,85 @@ private static StreamingChatMessageContent GenerateStreamingMessageContent(strin
665681
return content.Items.Count > 0 ? content : null;
666682
}
667683

668-
private static AnnotationContent GenerateAnnotationContent(MessageTextAnnotation annotation)
684+
private static AnnotationContent? GenerateAnnotationContent(MessageTextAnnotation annotation)
669685
{
670-
string? fileId = null;
671-
672686
if (annotation is MessageTextFileCitationAnnotation fileCitationAnnotation)
673687
{
674-
fileId = fileCitationAnnotation.FileId;
688+
return
689+
new AnnotationContent(
690+
kind: AnnotationKind.FileCitation,
691+
label: annotation.Text,
692+
referenceId: fileCitationAnnotation.FileId)
693+
{
694+
InnerContent = annotation,
695+
StartIndex = fileCitationAnnotation.StartIndex,
696+
EndIndex = fileCitationAnnotation.EndIndex,
697+
};
698+
}
699+
if (annotation is MessageTextUrlCitationAnnotation urlCitationAnnotation)
700+
{
701+
return
702+
new AnnotationContent(
703+
kind: AnnotationKind.UrlCitation,
704+
label: annotation.Text,
705+
referenceId: urlCitationAnnotation.UrlCitation.Url)
706+
{
707+
InnerContent = annotation,
708+
Title = urlCitationAnnotation.UrlCitation.Title,
709+
StartIndex = urlCitationAnnotation.StartIndex,
710+
EndIndex = urlCitationAnnotation.EndIndex,
711+
};
675712
}
676713
else if (annotation is MessageTextFilePathAnnotation filePathAnnotation)
677714
{
678-
fileId = filePathAnnotation.FileId;
715+
return
716+
new AnnotationContent(
717+
label: annotation.Text,
718+
kind: AnnotationKind.TextCitation,
719+
referenceId: filePathAnnotation.FileId)
720+
{
721+
InnerContent = annotation,
722+
StartIndex = filePathAnnotation.StartIndex,
723+
EndIndex = filePathAnnotation.EndIndex,
724+
};
679725
}
680726

681-
return
682-
new(annotation.Text)
683-
{
684-
Quote = annotation.Text,
685-
FileId = fileId,
686-
};
727+
return null;
687728
}
688729

689-
private static StreamingAnnotationContent GenerateStreamingAnnotationContent(TextAnnotationUpdate annotation)
730+
private static StreamingAnnotationContent? GenerateStreamingAnnotationContent(TextAnnotationUpdate annotation)
690731
{
691-
string? fileId = null;
732+
string? referenceId = null;
733+
AnnotationKind kind;
692734

693735
if (!string.IsNullOrEmpty(annotation.OutputFileId))
694736
{
695-
fileId = annotation.OutputFileId;
737+
referenceId = annotation.OutputFileId;
738+
kind = AnnotationKind.TextCitation;
696739
}
697740
else if (!string.IsNullOrEmpty(annotation.InputFileId))
698741
{
699-
fileId = annotation.InputFileId;
742+
referenceId = annotation.InputFileId;
743+
kind = AnnotationKind.FileCitation;
744+
}
745+
else if (!string.IsNullOrEmpty(annotation.Url))
746+
{
747+
referenceId = annotation.Url;
748+
kind = AnnotationKind.UrlCitation;
749+
}
750+
else
751+
{
752+
return null;
700753
}
701754

702755
return
703-
new(annotation.TextToReplace)
756+
new StreamingAnnotationContent(kind, referenceId)
704757
{
705-
StartIndex = annotation.StartIndex ?? 0,
706-
EndIndex = annotation.EndIndex ?? 0,
707-
FileId = fileId,
758+
Label = annotation.TextToReplace,
759+
InnerContent = annotation,
760+
Title = annotation.Title,
761+
StartIndex = annotation.StartIndex,
762+
EndIndex = annotation.EndIndex,
708763
};
709764
}
710765

‎dotnet/src/Agents/AzureAI/Logging/AgentThreadActionsLogMessages.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// Copyright (c) Microsoft. All rights reserved.
2+
using System;
23
using System.Diagnostics.CodeAnalysis;
34
using Azure.AI.Projects;
45
using Microsoft.Extensions.Logging;
@@ -136,4 +137,18 @@ public static partial void LogAzureAIAgentPolledRunStatus(
136137
RunStatus runStatus,
137138
string runId,
138139
string threadId);
140+
141+
/// <summary>
142+
/// Logs <see cref="AgentThreadActions"/> polled run status (complete).
143+
/// </summary>
144+
[LoggerMessage(
145+
EventId = 0,
146+
Level = LogLevel.Warning,
147+
Message = "[{MethodName}] Unknown annotation '{Type}': {RunId}/{ThreadId}.")]
148+
public static partial void LogAzureAIAgentUnknownAnnotation(
149+
this ILogger logger,
150+
string methodName,
151+
string runId,
152+
string threadId,
153+
Type type);
139154
}

0 commit comments

Comments
 (0)