Python: Feature python vector stores preb (#12271)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->
Overhaul of the VectorStores to get it matching with dotnet and the
latest development there.

Closes: 
- #11938 
- #11598 
- #11597
- #11517 
- #10561 
- #10391 
- #9911 
- #9892 
- #10867 

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

Many changes, some highlights:
- Moved from different vector fields types, to a single with a enum
first VectorStoreField()
- Renamed several items
- Made embeddinggenerrator a part of the vector field spec and do
automatic vectorization from there or from the same param on a
collection
- Adds hybrid search using the same setup
- Removed the intermediate TextSeachVectorSearch object need, by
allowing `create_search_function` directly on a collection

Will write out full changes in a migration guide in the docs.

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄

Author: eavanvalkenburg

python/.coveragerc

@@ -1,20 +1,7 @@
 [run]
 source = semantic_kernel
 omit = 
-    semantic_kernel/connectors/memory/astradb/*
-    semantic_kernel/connectors/memory/azure_cognitive_search/*
-    semantic_kernel/connectors/memory/azure_cosmosdb/*
-    semantic_kernel/connectors/memory/azure_cosmosdb_no_sql/*
-    semantic_kernel/connectors/memory/chroma/chroma_memory_store.py
-    semantic_kernel/connectors/memory/milvus/*
-    semantic_kernel/connectors/memory/mongodb_atlas/mongodb_atlas_memory_store.py
-    semantic_kernel/connectors/memory/pinecone/pinecone_memory_store.py
-    semantic_kernel/connectors/memory/pinecone/utils.py
-    semantic_kernel/connectors/memory/postgres/postgres_memory_store.py
-    semantic_kernel/connectors/memory/qdrant/qdrant_memory_store.py
-    semantic_kernel/connectors/memory/redis/redis_memory_store.py
-    semantic_kernel/connectors/memory/usearch/*
-    semantic_kernel/connectors/memory/weaviate/weaviate_memory_store.py
+    semantic_kernel/connectors/memory_stores/*
     semantic_kernel/reliability/*
     semantic_kernel/memory/*
 

python/mypy.ini

@@ -17,50 +17,9 @@ disable_error_code = method-assign
 
 [mypy-semantic_kernel.memory.*]
 ignore_errors = true
-# TODO (eavanvalkenburg): remove this
-# https://github.com/microsoft/semantic-kernel/issues/6463
+# TODO (eavanvalkenburg): remove this when removing the memory stores
 
-[mypy-semantic_kernel.connectors.memory.astradb.*]
+[mypy-semantic_kernel.connectors.memory_stores.*]
 ignore_errors = true
+# TODO (eavanvalkenburg): remove this when removing the memory stores
 
-[mypy-semantic_kernel.connectors.memory.azure_ai_search.*]
-ignore_errors = false
-[mypy-semantic_kernel.connectors.memory.azure_cognitive_search.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.azure_cosmosdb.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.azure_cosmosdb_no_sql.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.chroma.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.milvus.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.mongodb_atlas.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.pinecone.pinecone_memory_store]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.postgres.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.qdrant.qdrant_vector_record_store.*]
-ignore_errors = true
-[mypy-semantic_kernel.connectors.memory.qdrant.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.redis.redis_vector_record_store.*]
-ignore_errors = true
-[mypy-semantic_kernel.connectors.memory.redis.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.usearch.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.weaviate.*]
-ignore_errors = true

python/pyproject.toml

@@ -123,7 +123,7 @@ pandas = [
     "pandas ~= 2.2"
 ]
 pinecone = [
-    "pinecone[asyncio, grpc] ~= 6.0"
+    "pinecone[asyncio, grpc] ~= 7.0"
 ]
 postgres = [
     "psycopg[binary,pool] ~= 3.2"
@@ -132,7 +132,7 @@ qdrant = [
     "qdrant-client ~= 1.9"
 ]
 redis = [
-    "redis[hiredis] >= 5,< 6",
+    "redis[hiredis] ~= 6.0",
     "types-redis ~= 4.6.0.20240425",
     "redisvl ~= 0.4"
 ]
@@ -215,6 +215,7 @@ select = [
 ignore = [
     "D100", #allow missing docstring in public module
     "D104", #allow missing docstring in public package
+    "D418", #allow docstring on overloaded function
     "TD003", #allow missing link to todo issue
     "FIX002" #allow todo
 ]

python/samples/concepts/caching/semantic_caching.py

@@ -8,36 +8,25 @@
 from uuid import uuid4
 
 from semantic_kernel import Kernel
-from semantic_kernel.connectors.ai.embedding_generator_base import EmbeddingGeneratorBase
 from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion, OpenAITextEmbedding
-from semantic_kernel.connectors.memory.in_memory.in_memory_store import InMemoryVectorStore
-from semantic_kernel.data import (
-    VectorizedSearchMixin,
-    VectorSearchOptions,
-    VectorStore,
-    VectorStoreRecordCollection,
-    VectorStoreRecordDataField,
-    VectorStoreRecordKeyField,
-    VectorStoreRecordVectorField,
-    vectorstoremodel,
-)
+from semantic_kernel.connectors.in_memory import InMemoryStore
+from semantic_kernel.data.vector import VectorStore, VectorStoreCollection, VectorStoreField, vectorstoremodel
 from semantic_kernel.filters import FilterTypes, FunctionInvocationContext, PromptRenderContext
 from semantic_kernel.functions import FunctionResult
 
 COLLECTION_NAME = "llm_responses"
 RECORD_ID_KEY = "cache_record_id"
 
 
-# Define a simple data model to store, the prompt, the result, and the prompt embedding.
-@vectorstoremodel
+# Define a simple data model to store, the prompt and the result
+# we annotate the prompt field as the vector field, the prompt itself will not be stored.
+# and if you use `include_vectors` in the search, it will return the vector, but not the prompt.
+@vectorstoremodel(collection_name=COLLECTION_NAME)
 @dataclass
 class CacheRecord:
-    prompt: Annotated[str, VectorStoreRecordDataField(embedding_property_name="prompt_embedding")]
-    result: Annotated[str, VectorStoreRecordDataField(is_full_text_searchable=True)]
-    prompt_embedding: Annotated[list[float], VectorStoreRecordVectorField(dimensions=1536)] = field(
-        default_factory=list
-    )
-    id: Annotated[str, VectorStoreRecordKeyField] = field(default_factory=lambda: str(uuid4()))
+    result: Annotated[str, VectorStoreField("data", is_full_text_indexed=True)]
+    prompt: Annotated[str | None, VectorStoreField("vector", dimensions=1536)] = None
+    id: Annotated[str, VectorStoreField("key")] = field(default_factory=lambda: str(uuid4()))
 
 
 # Define the filters, one for caching the results and one for using the cache.
@@ -46,16 +35,13 @@ class PromptCacheFilter:
 
     def __init__(
         self,
-        embedding_service: EmbeddingGeneratorBase,
         vector_store: VectorStore,
-        collection_name: str = COLLECTION_NAME,
         score_threshold: float = 0.2,
     ):
-        self.embedding_service = embedding_service
+        if vector_store.embedding_generator is None:
+            raise ValueError("The vector store must have an embedding generator.")
         self.vector_store = vector_store
-        self.collection: VectorStoreRecordCollection[str, CacheRecord] = vector_store.get_collection(
-            collection_name, data_model_type=CacheRecord
-        )
+        self.collection: VectorStoreCollection[str, CacheRecord] = vector_store.get_collection(record_type=CacheRecord)
         self.score_threshold = score_threshold
 
     async def on_prompt_render(
@@ -69,15 +55,10 @@ async def on_prompt_render(
         closer the match.
         """
         await next(context)
-        assert context.rendered_prompt  # nosec
-        prompt_embedding = await self.embedding_service.generate_raw_embeddings([context.rendered_prompt])
-        await self.collection.create_collection_if_not_exists()
-        assert isinstance(self.collection, VectorizedSearchMixin)  # nosec
-        results = await self.collection.vectorized_search(
-            vector=prompt_embedding[0], options=VectorSearchOptions(vector_field_name="prompt_embedding", top=1)
-        )
+        await self.collection.ensure_collection_exists()
+        results = await self.collection.search(context.rendered_prompt, vector_property_name="prompt", top=1)
         async for result in results.results:
-            if result.score < self.score_threshold:
+            if result.score and result.score < self.score_threshold:
                 context.function_result = FunctionResult(
                     function=context.function.metadata,
                     value=result.record.result,
@@ -92,13 +73,8 @@ async def on_function_invocation(
         await next(context)
         result = context.result
         if result and result.rendered_prompt and RECORD_ID_KEY not in result.metadata:
-            prompt_embedding = await self.embedding_service.generate_embeddings([result.rendered_prompt])
-            cache_record = CacheRecord(
-                prompt=result.rendered_prompt,
-                result=str(result),
-                prompt_embedding=prompt_embedding[0],
-            )
-            await self.collection.create_collection_if_not_exists()
+            cache_record = CacheRecord(prompt=result.rendered_prompt, result=str(result))
+            await self.collection.ensure_collection_exists()
             await self.collection.upsert(cache_record)
 
 
@@ -118,11 +94,10 @@ async def main():
     chat = OpenAIChatCompletion(service_id="default")
     embedding = OpenAITextEmbedding(service_id="embedder")
     kernel.add_service(chat)
-    kernel.add_service(embedding)
     # create the in-memory vector store
-    vector_store = InMemoryVectorStore()
+    vector_store = InMemoryStore(embedding_generator=embedding)
     # create the cache filter and add the filters to the kernel
-    cache = PromptCacheFilter(embedding_service=embedding, vector_store=vector_store)
+    cache = PromptCacheFilter(vector_store=vector_store)
     kernel.add_filter(FilterTypes.PROMPT_RENDERING, cache.on_prompt_render)
     kernel.add_filter(FilterTypes.FUNCTION_INVOCATION, cache.on_function_invocation)
 

python/samples/concepts/chat_history/store_chat_history_in_cosmosdb.py

@@ -7,17 +7,11 @@
 from samples.concepts.setup.chat_completion_services import Services, get_chat_completion_service_and_request_settings
 from semantic_kernel import Kernel
 from semantic_kernel.connectors.ai import FunctionChoiceBehavior
-from semantic_kernel.connectors.memory.azure_cosmos_db.azure_cosmos_db_no_sql_store import AzureCosmosDBNoSQLStore
+from semantic_kernel.connectors.azure_cosmos_db import CosmosNoSqlStore
 from semantic_kernel.contents import ChatHistory, ChatMessageContent
 from semantic_kernel.core_plugins.math_plugin import MathPlugin
 from semantic_kernel.core_plugins.time_plugin import TimePlugin
-from semantic_kernel.data import (
-    VectorStore,
-    VectorStoreRecordCollection,
-    VectorStoreRecordDataField,
-    VectorStoreRecordKeyField,
-    vectorstoremodel,
-)
+from semantic_kernel.data.vector import VectorStore, VectorStoreCollection, VectorStoreField, vectorstoremodel
 
 """
 This sample demonstrates how to build a conversational chatbot
@@ -39,9 +33,9 @@
 @vectorstoremodel
 @dataclass
 class ChatHistoryModel:
-    session_id: Annotated[str, VectorStoreRecordKeyField]
-    user_id: Annotated[str, VectorStoreRecordDataField(is_filterable=True)]
-    messages: Annotated[list[dict[str, str]], VectorStoreRecordDataField(is_filterable=True)]
+    session_id: Annotated[str, VectorStoreField("key")]
+    user_id: Annotated[str, VectorStoreField("data", is_indexed=True)]
+    messages: Annotated[list[dict[str, str]], VectorStoreField("data", is_indexed=True)]
 
 
 # 2. We then create a class that extends the ChatHistory class
@@ -55,7 +49,7 @@ class ChatHistoryInCosmosDB(ChatHistory):
     session_id: str
     user_id: str
     store: VectorStore
-    collection: VectorStoreRecordCollection[str, ChatHistoryModel] | None = None
+    collection: VectorStoreCollection[str, ChatHistoryModel] | None = None
 
     async def create_collection(self, collection_name: str) -> None:
         """Create a collection with the inbuild data model using the vector store.
@@ -64,9 +58,9 @@ async def create_collection(self, collection_name: str) -> None:
         """
         self.collection = self.store.get_collection(
             collection_name=collection_name,
-            data_model_type=ChatHistoryModel,
+            record_type=ChatHistoryModel,
         )
-        await self.collection.create_collection_if_not_exists()
+        await self.collection.ensure_collection_exists()
 
     async def store_messages(self) -> None:
         """Store the chat history in the Cosmos DB.
@@ -175,7 +169,7 @@ async def main() -> None:
 
     # First we enter the store context manager to connect.
     # The create_database flag will create the database if it does not exist.
-    async with AzureCosmosDBNoSQLStore(create_database=True) as store:
+    async with CosmosNoSqlStore(create_database=True) as store:
         # Then we create the chat history in CosmosDB.
         history = ChatHistoryInCosmosDB(store=store, session_id=session_id, user_id="user")
         # Finally we create the collection.
@@ -191,7 +185,7 @@ async def main() -> None:
         except Exception:
             print("Closing chat...")
         if delete_when_done and history.collection:
-            await history.collection.delete_collection()
+            await history.collection.ensure_collection_deleted()
 
 
 if __name__ == "__main__":

python/samples/concepts/memory/azure_ai_search_hotel_samples/1_interact_with_the_collection.py

@@ -0,0 +1,73 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+
+from samples.concepts.memory.azure_ai_search_hotel_samples.data_model import (
+    HotelSampleClass,
+    custom_index,
+    load_records,
+)
+from semantic_kernel.connectors.ai.open_ai import OpenAITextEmbedding
+from semantic_kernel.connectors.azure_ai_search import AzureAISearchCollection
+
+"""
+With the data model and records defined in data_model.py, this script will create an Azure AI Search collection,
+upsert the records, and then search the collection using vector and hybrid search.
+The script will print the first five records in the collection and the search results.
+The script will also delete the collection at the end.
+
+Note that we add the OpenAITextEmbedding to the collection, which is used to generate the vectors.
+To use the built-in embedding in Azure AI Search, remove this and add that definition to the custom_index.
+"""
+
+
+async def main(query: str):
+    records = load_records()
+    # Create the Azure AI Search collection
+    async with AzureAISearchCollection[str, HotelSampleClass](
+        record_type=HotelSampleClass, embedding_generator=OpenAITextEmbedding()
+    ) as collection:
+        # Check if the collection exists.
+        await collection.ensure_collection_exists(index=custom_index)
+        await collection.upsert(records)
+        # get the first five records to check the upsert worked.
+        results = await collection.get(order_by="HotelName", top=5)
+        print("Get first five records: ")
+        if results:
+            for result in results:
+                print(
+                    f"    {result.HotelId} (in {result.Address.City}, {result.Address.Country}): {result.Description}"
+                )
+
+        print("\n")
+        print("Search results using vector: ")
+        # Use search to search using the vector.
+        results = await collection.search(
+            query,
+            vector_property_name="DescriptionVector",
+        )
+        async for result in results.results:
+            print(
+                f"    {result.record.HotelId} (in {result.record.Address.City}, "
+                f"{result.record.Address.Country}): {result.record.Description} (score: {result.score})"
+            )
+        print("\n")
+        print("Search results using hybrid: ")
+        # Use hybrid search to search using the vector.
+        results = await collection.hybrid_search(
+            query,
+            vector_property_name="DescriptionVector",
+            additional_property_name="Description",
+        )
+        async for result in results.results:
+            print(
+                f"    {result.record.HotelId} (in {result.record.Address.City}, "
+                f"{result.record.Address.Country}): {result.record.Description} (score: {result.score})"
+            )
+
+        await collection.ensure_collection_deleted()
+
+
+if __name__ == "__main__":
+    query = "swimming pool and good internet connection"
+    asyncio.run(main(query=query))