From 1d37252bf36eb39780bf388d355961e018c211f5 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Wed, 30 Jul 2025 21:16:47 -0700 Subject: [PATCH 1/3] Magentic orchestration to return the last agent message when limits reached --- .../Agents/Magentic/MagenticManagerActor.cs | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/dotnet/src/Agents/Magentic/MagenticManagerActor.cs b/dotnet/src/Agents/Magentic/MagenticManagerActor.cs index 881dfc365110..025d611de65e 100644 --- a/dotnet/src/Agents/Magentic/MagenticManagerActor.cs +++ b/dotnet/src/Agents/Magentic/MagenticManagerActor.cs @@ -141,7 +141,16 @@ private async ValueTask ManageAsync(CancellationToken cancellationToken) if (this._invocationCount >= this._manager.MaximumInvocationCount) { - await this.PublishMessageAsync("Maximum number of invocations reached.".AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false); + this.Logger.LogMagenticManagerTaskFailed(this.Context.Topic); + try + { + var partialResult = this._chat.Last((message) => message.Role == AuthorRole.Assistant); + await this.PublishMessageAsync(partialResult.AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false); + } + catch (InvalidOperationException) + { + await this.PublishMessageAsync("I've reaches the maximum number of invocations. No partial result available.".AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false); + } break; } @@ -157,7 +166,15 @@ private async ValueTask ManageAsync(CancellationToken cancellationToken) if (this._retryCount >= this._manager.MaximumResetCount) { this.Logger.LogMagenticManagerTaskFailed(this.Context.Topic); - await this.PublishMessageAsync("I've experienced multiple failures and am unable to continue.".AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false); + try + { + var partialResult = this._chat.Last((message) => message.Role == AuthorRole.Assistant); + await this.PublishMessageAsync(partialResult.AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false); + } + catch (InvalidOperationException) + { + await this.PublishMessageAsync("I've experienced multiple failures and am unable to continue. No partial result available.".AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false); + } break; } From 0054c88950c9cb4e68a49b093c971343a7fbba1f Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Sat, 2 Aug 2025 22:19:03 -0700 Subject: [PATCH 2/3] Add unit tests --- .../Magentic/MagenticOrchestrationTests.cs | 236 ++++++++++++++++++ 1 file changed, 236 insertions(+) diff --git a/dotnet/src/Agents/UnitTests/Magentic/MagenticOrchestrationTests.cs b/dotnet/src/Agents/UnitTests/Magentic/MagenticOrchestrationTests.cs index 036edf217dcf..516380c9895c 100644 --- a/dotnet/src/Agents/UnitTests/Magentic/MagenticOrchestrationTests.cs +++ b/dotnet/src/Agents/UnitTests/Magentic/MagenticOrchestrationTests.cs @@ -55,6 +55,216 @@ public async Task MagenticOrchestrationWithMultipleAgentsAsync() Assert.Equal(0, mockAgent3.InvokeCount); } + [Fact] + public async Task MagenticOrchestrationMaxInvocationCountReached_WithoutPartialResultAsync() + { + // Arrange + await using InProcessRuntime runtime = new(); + + MockAgent mockAgent1 = CreateMockAgent(1, "abc"); + MockAgent mockAgent2 = CreateMockAgent(2, "xyz"); + MockAgent mockAgent3 = CreateMockAgent(3, "lmn"); + + string jsonStatus = + $$""" + { + "Name": "{{mockAgent1.Name}}", + "Instruction":"Proceed", + "Reason":"TestReason", + "IsTaskComplete": { + "Result": false, + "Reason": "Test" + }, + "IsTaskProgressing": { + "Result": true, + "Reason": "Test" + }, + "IsTaskInLoop": { + "Result": false, + "Reason": "Test" + } + } + """; + Mock chatServiceMock = CreateMockChatCompletionService(jsonStatus); + + FakePromptExecutionSettings settings = new(); + StandardMagenticManager manager = new(chatServiceMock.Object, settings) + { + MaximumInvocationCount = 1, // Fast failure for testing + }; + + MagenticOrchestration orchestration = new(manager, [mockAgent1, mockAgent2, mockAgent3]); + + // Act + await runtime.StartAsync(); + + const string InitialInput = "123"; + OrchestrationResult result = await orchestration.InvokeAsync(InitialInput, runtime); + string response = await result.GetValueAsync(TimeSpan.FromSeconds(20)); + + // Assert + Assert.NotNull(response); + Assert.Contains("No partial result available.", response); + } + + [Fact] + public async Task MagenticOrchestrationMaxInvocationCountReached_WithPartialResultAsync() + { + // Arrange + await using InProcessRuntime runtime = new(); + + MockAgent mockAgent1 = CreateMockAgent(1, "abc"); + MockAgent mockAgent2 = CreateMockAgent(2, "xyz"); + MockAgent mockAgent3 = CreateMockAgent(3, "lmn"); + + string jsonStatus = + $$""" + { + "Name": "{{mockAgent1.Name}}", + "Instruction":"Proceed", + "Reason":"TestReason", + "IsTaskComplete": { + "Result": false, + "Reason": "Test" + }, + "IsTaskProgressing": { + "Result": true, + "Reason": "Test" + }, + "IsTaskInLoop": { + "Result": false, + "Reason": "Test" + } + } + """; + Mock chatServiceMock = CreateMockChatCompletionService(jsonStatus); + + FakePromptExecutionSettings settings = new(); + StandardMagenticManager manager = new(chatServiceMock.Object, settings) + { + MaximumInvocationCount = 2, // Fast failure for testing but at least one invocation + }; + + MagenticOrchestration orchestration = new(manager, [mockAgent1, mockAgent2, mockAgent3]); + + // Act + await runtime.StartAsync(); + + const string InitialInput = "123"; + OrchestrationResult result = await orchestration.InvokeAsync(InitialInput, runtime); + string response = await result.GetValueAsync(TimeSpan.FromSeconds(20)); + + // Assert + Assert.NotNull(response); + Assert.Equal("abc", response); + } + + [Fact] + public async Task MagenticOrchestrationMaxResetCountReached_WithoutPartialResultAsync() + { + // Arrange + await using InProcessRuntime runtime = new(); + + MockAgent mockAgent1 = CreateMockAgent(1, "abc"); + MockAgent mockAgent2 = CreateMockAgent(2, "xyz"); + MockAgent mockAgent3 = CreateMockAgent(3, "lmn"); + + string jsonStatus = + $$""" + { + "Name": "{{mockAgent1.Name}}", + "Instruction":"Proceed", + "Reason":"TestReason", + "IsTaskComplete": { + "Result": false, + "Reason": "Test" + }, + "IsTaskProgressing": { + "Result": false, + "Reason": "Test" + }, + "IsTaskInLoop": { + "Result": true, + "Reason": "Test" + } + } + """; + Mock chatServiceMock = CreateMockChatCompletionService(jsonStatus); + + FakePromptExecutionSettings settings = new(); + StandardMagenticManager manager = new(chatServiceMock.Object, settings) + { + MaximumResetCount = 1, // Fast failure for testing + MaximumStallCount = 0, // No stalls allowed + }; + + MagenticOrchestration orchestration = new(manager, [mockAgent1, mockAgent2, mockAgent3]); + + // Act + await runtime.StartAsync(); + + const string InitialInput = "123"; + OrchestrationResult result = await orchestration.InvokeAsync(InitialInput, runtime); + string response = await result.GetValueAsync(TimeSpan.FromSeconds(20)); + + // Assert + Assert.NotNull(response); + Assert.Contains("No partial result available.", response); + } + + [Fact] + public async Task MagenticOrchestrationMaxResetCountReached_WithPartialResultAsync() + { + // Arrange + await using InProcessRuntime runtime = new(); + + MockAgent mockAgent1 = CreateMockAgent(1, "abc"); + MockAgent mockAgent2 = CreateMockAgent(2, "xyz"); + MockAgent mockAgent3 = CreateMockAgent(3, "lmn"); + + string jsonStatus = + $$""" + { + "Name": "{{mockAgent1.Name}}", + "Instruction":"Proceed", + "Reason":"TestReason", + "IsTaskComplete": { + "Result": false, + "Reason": "Test" + }, + "IsTaskProgressing": { + "Result": false, + "Reason": "Test" + }, + "IsTaskInLoop": { + "Result": true, + "Reason": "Test" + } + } + """; + Mock chatServiceMock = CreateMockChatCompletionService(jsonStatus); + + FakePromptExecutionSettings settings = new(); + StandardMagenticManager manager = new(chatServiceMock.Object, settings) + { + MaximumResetCount = 1, // Fast failure for testing but at least one response + MaximumStallCount = 2, + }; + + MagenticOrchestration orchestration = new(manager, [mockAgent1, mockAgent2, mockAgent3]); + + // Act + await runtime.StartAsync(); + + const string InitialInput = "123"; + OrchestrationResult result = await orchestration.InvokeAsync(InitialInput, runtime); + string response = await result.GetValueAsync(TimeSpan.FromSeconds(20)); + + // Assert + Assert.NotNull(response); + Assert.Contains("abc", response); + } + private async Task ExecuteOrchestrationAsync(InProcessRuntime runtime, string answer, params Agent[] mockAgents) { // Act @@ -81,6 +291,7 @@ private static MockAgent CreateMockAgent(int index, string response) { return new() { + Name = $"MockAgent{index}", Description = $"test {index}", Response = [new(AuthorRole.Assistant, response)] }; @@ -130,4 +341,29 @@ MagenticProgressLedger CreateLedger(bool isTaskComplete, string name) } } } + + private static Mock CreateMockChatCompletionService(string response) + { + Mock chatServiceMock = new(MockBehavior.Strict); + + chatServiceMock.Setup( + (service) => service.GetChatMessageContentsAsync( + It.IsAny(), + It.IsAny(), + null, + It.IsAny())) + .ReturnsAsync([new ChatMessageContent(AuthorRole.Assistant, response)]); + + return chatServiceMock; + } + + private sealed class FakePromptExecutionSettings : PromptExecutionSettings + { + public override PromptExecutionSettings Clone() + { + return this; + } + + public object? ResponseFormat { get; set; } + } } From 5e10f4175d12feb0d9a8ec23e88122889f98cedc Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Sun, 3 Aug 2025 14:01:05 -0700 Subject: [PATCH 3/3] Minor improvement --- .../src/Agents/UnitTests/Magentic/MagenticOrchestrationTests.cs | 2 +- python/semantic_kernel/agents/orchestration/magentic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dotnet/src/Agents/UnitTests/Magentic/MagenticOrchestrationTests.cs b/dotnet/src/Agents/UnitTests/Magentic/MagenticOrchestrationTests.cs index 516380c9895c..c6d3227690dd 100644 --- a/dotnet/src/Agents/UnitTests/Magentic/MagenticOrchestrationTests.cs +++ b/dotnet/src/Agents/UnitTests/Magentic/MagenticOrchestrationTests.cs @@ -248,7 +248,7 @@ public async Task MagenticOrchestrationMaxResetCountReached_WithPartialResultAsy StandardMagenticManager manager = new(chatServiceMock.Object, settings) { MaximumResetCount = 1, // Fast failure for testing but at least one response - MaximumStallCount = 2, + MaximumStallCount = 2, // Allow some stalls for at least one response }; MagenticOrchestration orchestration = new(manager, [mockAgent1, mockAgent2, mockAgent3]); diff --git a/python/semantic_kernel/agents/orchestration/magentic.py b/python/semantic_kernel/agents/orchestration/magentic.py index 45b8e29ec89c..4c4eae440f1e 100644 --- a/python/semantic_kernel/agents/orchestration/magentic.py +++ b/python/semantic_kernel/agents/orchestration/magentic.py @@ -660,7 +660,7 @@ async def _check_within_limits(self) -> bool: if hit_round_limit or hit_reset_limit: limit_type = "round" if hit_round_limit else "reset" - logger.debug(f"Max {limit_type} count reached.") + logger.error(f"Max {limit_type} count reached.") # Retrieve the latest assistant content produced so far partial_result = next(