.Net: Support for configuring dimensions in Google AI embeddings generation

dotnet/Directory.Packages.props

@@ -24,6 +24,7 @@
     <PackageVersion Include="Dapr.Actors.AspNetCore" Version="1.14.0" />
     <PackageVersion Include="Dapr.AspNetCore" Version="1.14.0" />
     <PackageVersion Include="FastBertTokenizer" Version="1.0.28" />
+    <PackageVersion Include="Google.Apis.Auth" Version="1.69.0" />
     <PackageVersion Include="mcpdotnet" Version="1.0.1.3" />
     <PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.13" />
     <PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="8.0.13" />
@@ -142,9 +143,9 @@
     <PackageVersion Include="Milvus.Client" Version="2.3.0-preview.1" />
     <PackageVersion Include="Testcontainers" Version="4.1.0" />
     <PackageVersion Include="Testcontainers.Milvus" Version="4.1.0" />
-    <PackageVersion Include="Testcontainers.MongoDB" Version="4.1.0"/>
-    <PackageVersion Include="Testcontainers.PostgreSql" Version="4.1.0"/>
-    <PackageVersion Include="Testcontainers.Redis" Version="4.1.0"/>
+    <PackageVersion Include="Testcontainers.MongoDB" Version="4.1.0" />
+    <PackageVersion Include="Testcontainers.PostgreSql" Version="4.1.0" />
+    <PackageVersion Include="Testcontainers.Redis" Version="4.1.0" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="5.2.2" />
     <PackageVersion Include="Qdrant.Client" Version="1.12.0" />
     <!-- Symbols -->

dotnet/samples/Concepts/Concepts.csproj

@@ -15,6 +15,7 @@
 
   <ItemGroup>
     <PackageReference Include="Docker.DotNet" />
+    <PackageReference Include="Google.Apis.Auth" />
     <PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
     <PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />

dotnet/samples/Concepts/Memory/Google_EmbeddingGeneration.cs

@@ -0,0 +1,120 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Google.Apis.Auth.OAuth2;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Embeddings;
+using xRetry;
+
+namespace Memory;
+
+// The following example shows how to use Semantic Kernel with Google AI and Google's Vertex AI for embedding generation,
+// including the ability to specify custom dimensions.
+public class Google_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
+{
+    /// <summary>
+    /// This test demonstrates how to use the Google Vertex AI embedding generation service with default dimensions.
+    /// </summary>
+    /// <remarks>
+    /// Currently custom dimensions are not supported for Vertex AI.
+    /// </remarks>
+    [RetryFact(typeof(HttpOperationException))]
+    public async Task GenerateEmbeddingWithDefaultDimensionsUsingVertexAI()
+    {
+        string? bearerToken = null;
+
+        Assert.NotNull(TestConfiguration.VertexAI.EmbeddingModelId);
+        Assert.NotNull(TestConfiguration.VertexAI.ClientId);
+        Assert.NotNull(TestConfiguration.VertexAI.ClientSecret);
+        Assert.NotNull(TestConfiguration.VertexAI.Location);
+        Assert.NotNull(TestConfiguration.VertexAI.ProjectId);
+
+        IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
+        kernelBuilder.AddVertexAIEmbeddingGeneration(
+                modelId: TestConfiguration.VertexAI.EmbeddingModelId!,
+                bearerTokenProvider: GetBearerToken,
+                location: TestConfiguration.VertexAI.Location,
+                projectId: TestConfiguration.VertexAI.ProjectId);
+        Kernel kernel = kernelBuilder.Build();
+
+        var embeddingGenerator = kernel.GetRequiredService<ITextEmbeddingGenerationService>();
+
+        // Generate embeddings with the default dimensions for the model
+        var embeddings = await embeddingGenerator.GenerateEmbeddingsAsync(
+            ["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
+
+        Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Length}' dimensions (default) for the provided text");
+
+        // Uses Google.Apis.Auth.OAuth2 to get the bearer token
+        async ValueTask<string> GetBearerToken()
+        {
+            if (!string.IsNullOrEmpty(bearerToken))
+            {
+                return bearerToken;
+            }
+
+            var credential = GoogleWebAuthorizationBroker.AuthorizeAsync(
+                new ClientSecrets
+                {
+                    ClientId = TestConfiguration.VertexAI.ClientId,
+                    ClientSecret = TestConfiguration.VertexAI.ClientSecret
+                },
+                ["https://www.googleapis.com/auth/cloud-platform"],
+                "user",
+                CancellationToken.None);
+
+            var userCredential = await credential.WaitAsync(CancellationToken.None);
+            bearerToken = userCredential.Token.AccessToken;
+
+            return bearerToken;
+        }
+    }
+
+    [RetryFact(typeof(HttpOperationException))]
+    public async Task GenerateEmbeddingWithDefaultDimensionsUsingGoogleAI()
+    {
+        Assert.NotNull(TestConfiguration.GoogleAI.EmbeddingModelId);
+        Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
+
+        IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
+        kernelBuilder.AddGoogleAIEmbeddingGeneration(
+                modelId: TestConfiguration.GoogleAI.EmbeddingModelId!,
+                apiKey: TestConfiguration.GoogleAI.ApiKey);
+        Kernel kernel = kernelBuilder.Build();
+
+        var embeddingGenerator = kernel.GetRequiredService<ITextEmbeddingGenerationService>();
+
+        // Generate embeddings with the default dimensions for the model
+        var embeddings = await embeddingGenerator.GenerateEmbeddingsAsync(
+            ["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
+
+        Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Length}' dimensions (default) for the provided text");
+    }
+
+    [RetryFact(typeof(HttpOperationException))]
+    public async Task GenerateEmbeddingWithCustomDimensionsUsingGoogleAI()
+    {
+        Assert.NotNull(TestConfiguration.GoogleAI.EmbeddingModelId);
+        Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
+
+        // Specify custom dimensions for the embeddings
+        const int CustomDimensions = 512;
+
+        IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
+        kernelBuilder.AddGoogleAIEmbeddingGeneration(
+                modelId: TestConfiguration.GoogleAI.EmbeddingModelId!,
+                apiKey: TestConfiguration.GoogleAI.ApiKey,
+                dimensions: CustomDimensions);
+        Kernel kernel = kernelBuilder.Build();
+
+        var embeddingGenerator = kernel.GetRequiredService<ITextEmbeddingGenerationService>();
+
+        // Generate embeddings with the specified custom dimensions
+        var embeddings = await embeddingGenerator.GenerateEmbeddingsAsync(
+            ["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
+
+        Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Length}' dimensions (custom: '{CustomDimensions}') for the provided text");
+
+        // Verify that we received embeddings with our requested dimensions
+        Assert.Equal(CustomDimensions, embeddings[0].Length);
+    }
+}

dotnet/src/Connectors/Connectors.Google.UnitTests/Core/GoogleAI/GoogleAIClientEmbeddingsGenerationTests.cs

@@ -142,14 +142,61 @@ public async Task ItCreatesPostRequestWithSemanticKernelVersionHeaderAsync()
         Assert.Equal(expectedVersion, header);
     }
 
+    [Fact]
+    public async Task ShouldIncludeDimensionsInAllRequestsAsync()
+    {
+        // Arrange
+        const int Dimensions = 512;
+        var client = this.CreateEmbeddingsClient(dimensions: Dimensions);
+        var dataToEmbed = new List<string>()
+        {
+            "First text to embed",
+            "Second text to embed",
+            "Third text to embed"
+        };
+
+        // Act
+        await client.GenerateEmbeddingsAsync(dataToEmbed);
+
+        // Assert
+        var request = JsonSerializer.Deserialize<GoogleAIEmbeddingRequest>(this._messageHandlerStub.RequestContent);
+        Assert.NotNull(request);
+        Assert.Equal(dataToEmbed.Count, request.Requests.Count);
+        Assert.All(request.Requests, item => Assert.Equal(Dimensions, item.Dimensions));
+    }
+
+    [Fact]
+    public async Task ShouldNotIncludeDimensionsInAllRequestsWhenNotProvidedAsync()
+    {
+        // Arrange
+        var client = this.CreateEmbeddingsClient();
+        var dataToEmbed = new List<string>()
+        {
+            "First text to embed",
+            "Second text to embed",
+            "Third text to embed"
+        };
+
+        // Act
+        await client.GenerateEmbeddingsAsync(dataToEmbed);
+
+        // Assert
+        var request = JsonSerializer.Deserialize<GoogleAIEmbeddingRequest>(this._messageHandlerStub.RequestContent);
+        Assert.NotNull(request);
+        Assert.Equal(dataToEmbed.Count, request.Requests.Count);
+        Assert.All(request.Requests, item => Assert.Null(item.Dimensions));
+    }
+
     private GoogleAIEmbeddingClient CreateEmbeddingsClient(
-        string modelId = "fake-model")
+        string modelId = "fake-model",
+        int? dimensions = null)
     {
         var client = new GoogleAIEmbeddingClient(
             httpClient: this._httpClient,
             modelId: modelId,
             apiVersion: GoogleAIVersion.V1,
-            apiKey: "fake-key");
+            apiKey: "fake-key",
+            dimensions: dimensions);
         return client;
     }
 

dotnet/src/Connectors/Connectors.Google.UnitTests/Core/GoogleAI/GoogleAIEmbeddingRequestTests.cs

@@ -1,41 +1,86 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System.Text.Json;
 using Microsoft.SemanticKernel.Connectors.Google.Core;
 using Xunit;
 
 namespace SemanticKernel.Connectors.Google.UnitTests.Core.GoogleAI;
 
 public sealed class GoogleAIEmbeddingRequestTests
 {
+    // Arrange
+    private static readonly string[] s_data = ["text1", "text2"];
+    private const string ModelId = "modelId";
+    private const string DimensionalityJsonPropertyName = "\"outputDimensionality\"";
+    private const int Dimensions = 512;
+
     [Fact]
     public void FromDataReturnsValidRequestWithData()
     {
-        // Arrange
-        string[] data = ["text1", "text2"];
-        var modelId = "modelId";
-
         // Act
-        var request = GoogleAIEmbeddingRequest.FromData(data, modelId);
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId);
 
         // Assert
         Assert.Equal(2, request.Requests.Count);
-        Assert.Equal(data[0], request.Requests[0].Content.Parts![0].Text);
-        Assert.Equal(data[1], request.Requests[1].Content.Parts![0].Text);
+        Assert.Equal(s_data[0], request.Requests[0].Content.Parts![0].Text);
+        Assert.Equal(s_data[1], request.Requests[1].Content.Parts![0].Text);
     }
 
     [Fact]
     public void FromDataReturnsValidRequestWithModelId()
     {
-        // Arrange
-        string[] data = ["text1", "text2"];
-        var modelId = "modelId";
+        // Act
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId);
+
+        // Assert
+        Assert.Equal(2, request.Requests.Count);
+        Assert.Equal($"models/{ModelId}", request.Requests[0].Model);
+        Assert.Equal($"models/{ModelId}", request.Requests[1].Model);
+    }
 
+    [Fact]
+    public void FromDataSetsDimensionsToNullWhenNotProvided()
+    {
         // Act
-        var request = GoogleAIEmbeddingRequest.FromData(data, modelId);
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId);
 
         // Assert
         Assert.Equal(2, request.Requests.Count);
-        Assert.Equal($"models/{modelId}", request.Requests[0].Model);
-        Assert.Equal($"models/{modelId}", request.Requests[1].Model);
+        Assert.Null(request.Requests[0].Dimensions);
+        Assert.Null(request.Requests[1].Dimensions);
+    }
+
+    [Fact]
+    public void FromDataJsonDoesNotIncludeDimensionsWhenNull()
+    {
+        // Act
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId);
+        string json = JsonSerializer.Serialize(request);
+
+        // Assert
+        Assert.DoesNotContain(DimensionalityJsonPropertyName, json);
+    }
+
+    [Fact]
+    public void FromDataSetsDimensionsWhenProvided()
+    {
+        // Act
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId, Dimensions);
+
+        // Assert
+        Assert.Equal(2, request.Requests.Count);
+        Assert.Equal(Dimensions, request.Requests[0].Dimensions);
+        Assert.Equal(Dimensions, request.Requests[1].Dimensions);
+    }
+
+    [Fact]
+    public void FromDataJsonIncludesDimensionsWhenProvided()
+    {
+        // Act
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId, Dimensions);
+        string json = JsonSerializer.Serialize(request);
+
+        // Assert
+        Assert.Contains($"{DimensionalityJsonPropertyName}:{Dimensions}", json);
     }
 }