Skip to content

Commit 4f0bf16

Browse files
SergeyMenshykhwestey-mrogerbarreto
authored
.Net: Contextual function selection to SK (#12321)
### Motivation and Context Adding contextual function selection functionality to SK. This allows AI agents to advertise functions relevant to the current context, rather than advertising all available functions. Contributes to: #10074 --------- Co-authored-by: westey <[email protected]> Co-authored-by: Roger Barreto <[email protected]>
1 parent cbf8fc2 commit 4f0bf16

File tree

12 files changed

+1369
-0
lines changed

12 files changed

+1369
-0
lines changed

‎dotnet/SK-dotnet.sln‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ EndProject
124124
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Linq", "Linq", "{B00AD427-0047-4850-BEF9-BA8237EA9D8B}"
125125
ProjectSection(SolutionItems) = preProject
126126
src\InternalUtilities\src\Linq\AsyncEnumerable.cs = src\InternalUtilities\src\Linq\AsyncEnumerable.cs
127+
src\InternalUtilities\src\Linq\EnumerableExtensions.cs = src\InternalUtilities\src\Linq\EnumerableExtensions.cs
127128
EndProjectSection
128129
EndProject
129130
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Http", "Http", "{1C19D805-3573-4477-BF07-40180FCDE1BD}"

‎dotnet/samples/Concepts/Agents/ChatCompletion_ContextualFunctionSelection.cs‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using Azure.AI.OpenAI;
4+
using Azure.Identity;
5+
using Microsoft.Extensions.AI;
6+
using Microsoft.SemanticKernel;
7+
using Microsoft.SemanticKernel.Agents;
8+
using Microsoft.SemanticKernel.Connectors.InMemory;
9+
using Microsoft.SemanticKernel.Functions;
10+
11+
namespace Agents;
12+
13+
#pragma warning disable SKEXP0130 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
14+
15+
/// <summary>
16+
/// Demonstrates the creation of a <see cref="ChatCompletionAgent"/> and adding capabilities
17+
/// for contextual function selection to it. Contextual function selection involves using
18+
/// Retrieval-Augmented Generation (RAG) to identify and select the most relevant functions
19+
/// based on the current context. The provider vectorizes the function names and descriptions,
20+
/// stores them in a specified vector store, and performs a vector search to find and provide
21+
/// the most pertinent functions to the AI model/agent for a given context.
22+
/// </summary>
23+
public class ChatCompletion_ContextualFunctionSelection(ITestOutputHelper output) : BaseTest(output)
24+
{
25+
/// <summary>
26+
/// Shows how to configure agent to use <see cref="ContextualFunctionProvider"/>
27+
/// to enable contextual function selection based on the current invocation context.
28+
/// </summary>
29+
[Fact]
30+
private async Task SelectFunctionsRelevantToCurrentInvocationContext()
31+
{
32+
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
33+
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
34+
.AsIEmbeddingGenerator(1536);
35+
36+
// Create our agent.
37+
Kernel kernel = this.CreateKernelWithChatCompletion();
38+
ChatCompletionAgent agent =
39+
new()
40+
{
41+
Name = "ReviewGuru",
42+
Instructions = "You are a friendly assistant that summarizes key points and sentiments from customer reviews. " +
43+
"For each response, list available functions",
44+
Kernel = kernel,
45+
Arguments = new(new PromptExecutionSettings { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(options: new FunctionChoiceBehaviorOptions { RetainArgumentTypes = true }) })
46+
};
47+
48+
// Create a thread and register context based function selection provider that will do RAG on
49+
// provided functions to advertise only those that are relevant to the current context.
50+
ChatHistoryAgentThread agentThread = new();
51+
52+
var allAvailableFunctions = GetAvailableFunctions();
53+
54+
agentThread.AIContextProviders.Add(
55+
new ContextualFunctionProvider(
56+
vectorStore: new InMemoryVectorStore(new InMemoryVectorStoreOptions() { EmbeddingGenerator = embeddingGenerator }),
57+
vectorDimensions: 1536,
58+
functions: allAvailableFunctions,
59+
maxNumberOfFunctions: 3, // Instruct the provider to return a maximum of 3 relevant functions
60+
loggerFactory: this.LoggerFactory
61+
)
62+
);
63+
64+
// Invoke and display assistant response
65+
ChatMessageContent message = await agent.InvokeAsync("Get and summarize customer review.", agentThread).FirstAsync();
66+
Console.WriteLine(message.Content);
67+
68+
//Expected output:
69+
/*
70+
Retrieves and summarizes customer reviews.
71+
72+
### Customer Reviews:
73+
1. **John D.** - ★★★★★
74+
*Comment:* Great product and fast shipping!
75+
*Date:* 2023-10-01
76+
2. **Jane S.** - ★★★★
77+
*Comment:* Good quality, but delivery was a bit slow.
78+
*Date:* 2023-09-28
79+
3. **Mike J.** - ★★★
80+
*Comment:* Average. Works as expected.
81+
*Date:* 2023-09-25
82+
83+
### Summary:
84+
The reviews indicate overall customer satisfaction, with highlights on product quality and shipping efficiency.
85+
While some customers experienced excellent service, others mentioned areas for improvement, particularly regarding delivery times.
86+
87+
If you need further analysis or insights, feel free to ask!
88+
89+
Available functions:
90+
- Tools-GetCustomerReviews
91+
- Tools-Summarize
92+
- Tools-CollectSentiments
93+
*/
94+
}
95+
96+
/// <summary>
97+
/// Shows how to configure agent to use <see cref="ContextualFunctionProvider"/>
98+
/// to enable contextual function selection based on the previous and current invocation context.
99+
/// </summary>
100+
[Fact]
101+
private async Task SelectFunctionsBasedOnPreviousAndCurrentInvocationContext()
102+
{
103+
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
104+
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
105+
.AsIEmbeddingGenerator(1536);
106+
107+
// Create our agent.
108+
Kernel kernel = this.CreateKernelWithChatCompletion();
109+
ChatCompletionAgent agent =
110+
new()
111+
{
112+
Name = "AzureAssistant",
113+
Instructions = "You are a helpful assistant that helps with Azure resource management. " +
114+
"Avoid including the phrase like 'If you need further assistance or have any additional tasks, feel free to let me know!' in any responses.",
115+
Kernel = kernel,
116+
Arguments = new(new PromptExecutionSettings { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(options: new FunctionChoiceBehaviorOptions { RetainArgumentTypes = true }) })
117+
};
118+
119+
// Create a thread and register context based function selection provider that will do RAG on
120+
// provided functions to advertise only those that are relevant to the current context.
121+
ChatHistoryAgentThread agentThread = new();
122+
123+
var allAvailableFunctions = GetAvailableFunctions();
124+
125+
agentThread.AIContextProviders.Add(
126+
new ContextualFunctionProvider(
127+
vectorStore: new InMemoryVectorStore(new InMemoryVectorStoreOptions() { EmbeddingGenerator = embeddingGenerator }),
128+
vectorDimensions: 1536,
129+
functions: allAvailableFunctions,
130+
maxNumberOfFunctions: 1, // Instruct the provider to return only one relevant function
131+
loggerFactory: this.LoggerFactory,
132+
options: new ContextualFunctionProviderOptions
133+
{
134+
NumberOfRecentMessagesInContext = 1 // Use only the last message from the previous agent invocation
135+
}
136+
)
137+
);
138+
139+
// Ask agent to provision a VM on Azure. The contextual function selection provider will return only one relevant function: `ProvisionVM`
140+
ChatMessageContent message = await agent.InvokeAsync("Please provision a VM on Azure", agentThread).FirstAsync();
141+
Console.WriteLine(message.Content);
142+
143+
//Expected output: "A virtual machine has been successfully provisioned on Azure with the ID: 7f2aa1e4-13ac-4875-9e63-278ee82f3729."
144+
145+
// Ask the agent to deploy the VM, intentionally referring to the VM as "it".
146+
// This demonstrates that the contextual function selection provider uses the last message from the previous invocation
147+
// to infer that the user is referring to the VM provisioned in the invocation and not any other Azure resource.
148+
// The provider will return only one relevant function to deploy the VM: `DeployVM`
149+
message = await agent.InvokeAsync("Deploy it", agentThread).FirstAsync();
150+
Console.WriteLine(message.Content);
151+
152+
//Expected output: "The virtual machine with ID: 7f2aa1e4-13ac-4875-9e63-278ee82f3729 has been successfully deployed."
153+
}
154+
155+
/// <summary>
156+
/// Returns a list of functions that belong to different categories.
157+
/// Some categories/functions are related to the prompt, while others
158+
/// are not. This is intentionally done to demonstrate the contextual
159+
/// function selection capabilities of the provider.
160+
/// </summary>
161+
private IReadOnlyList<AIFunction> GetAvailableFunctions()
162+
{
163+
List<AIFunction> reviewFunctions = [
164+
AIFunctionFactory.Create(() => """
165+
[
166+
{
167+
"reviewer": "John D.",
168+
"date": "2023-10-01",
169+
"rating": 5,
170+
"comment": "Great product and fast shipping!"
171+
},
172+
{
173+
"reviewer": "Jane S.",
174+
"date": "2023-09-28",
175+
"rating": 4,
176+
"comment": "Good quality, but delivery was a bit slow."
177+
},
178+
{
179+
"reviewer": "Mike J.",
180+
"date": "2023-09-25",
181+
"rating": 3,
182+
"comment": "Average. Works as expected."
183+
}
184+
]
185+
"""
186+
, "GetCustomerReviews"),
187+
];
188+
189+
List<AIFunction> sentimentFunctions = [
190+
AIFunctionFactory.Create((string text) => "The collected sentiment is mostly positive with a few neutral and negative opinions.", "CollectSentiments"),
191+
AIFunctionFactory.Create((string text) => "Sentiment trend identified: predominantly positive with increasing positive feedback.", "IdentifySentimentTrend"),
192+
];
193+
194+
List<AIFunction> summaryFunctions = [
195+
AIFunctionFactory.Create((string text) => "Summary generated based on input data: key points include market growth and customer satisfaction.", "Summarize"),
196+
AIFunctionFactory.Create((string text) => "Extracted themes: innovation, efficiency, customer satisfaction.", "ExtractThemes"),
197+
];
198+
199+
List<AIFunction> communicationFunctions = [
200+
AIFunctionFactory.Create((string address, string content) => "Email sent.", "SendEmail"),
201+
AIFunctionFactory.Create((string number, string text) => "Message sent.", "SendSms"),
202+
AIFunctionFactory.Create(() => "[email protected]", "MyEmail"),
203+
];
204+
205+
List<AIFunction> dateTimeFunctions = [
206+
AIFunctionFactory.Create(() => DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), "GetCurrentDateTime"),
207+
AIFunctionFactory.Create(() => DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss"), "GetCurrentUtcDateTime"),
208+
];
209+
210+
List<AIFunction> azureFunctions = [
211+
AIFunctionFactory.Create(() => $"Resource group provisioned: Id:{Guid.NewGuid()}", "ProvisionResourceGroup"),
212+
AIFunctionFactory.Create((Guid id) => $"Resource group deployed: Id:{id}", "DeployResourceGroup"),
213+
214+
AIFunctionFactory.Create(() => $"Storage account provisioned: Id:{Guid.NewGuid()}", "ProvisionStorageAccount"),
215+
AIFunctionFactory.Create((Guid id) => $"Storage account deployed: Id:{id}", "DeployStorageAccount"),
216+
217+
AIFunctionFactory.Create(() => $"VM provisioned: Id:{Guid.NewGuid()}", "ProvisionVM"),
218+
AIFunctionFactory.Create((Guid id) => $"VM deployed: Id:{id}", "DeployVM"),
219+
];
220+
221+
return [.. reviewFunctions, .. sentimentFunctions, .. summaryFunctions, .. communicationFunctions, .. dateTimeFunctions, .. azureFunctions];
222+
}
223+
}

‎dotnet/samples/Concepts/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom
2626
- [ChatCompletion_Rag: Shows how to easily add RAG to an agent](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/ChatCompletion_Rag.cs)
2727
- [ChatCompletion_Mem0: Shows how to add memory to an agent using mem0](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/ChatCompletion_Mem0.cs)
2828
- [ChatCompletion_Whiteboard: Shows how to add short term Whiteboarding memory to an agent](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/ChatCompletion_Whiteboard.cs)
29+
- [ChatCompletion_ContextualFunctionSelection: Shows how to add contextual function selection capabilities to an agent](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/ChatCompletion_ContextualFunctionSelection.cs)
2930

3031
### AudioToText - Different ways of using [`AudioToText`](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/AudioToText/IAudioToTextService.cs) services to extract text from audio
3132

0 commit comments

Comments
 (0)