Merge pull request #231 from poissoncorp/v7.0

RDBC-889 7.0 Python client with Vector API

Author: poissoncorp

.github/workflows/RavenClient.yml

@@ -2,9 +2,9 @@ name: tests/python
 
 on:
   push:
-    branches: [v5.2]
+    branches: [v7.0]
   pull_request:
-    branches: [v5.2]
+    branches: [v7.0]
   schedule:
     - cron: '0 10 * * *'
   workflow_dispatch:
@@ -29,8 +29,8 @@ jobs:
 
       strategy:
         matrix:
-          python-version: [ '3.8', '3.9', '3.10' ,'3.11', '3.12']
-          serverVersion: [ '5.4', '6.2', '7.0' ]
+          python-version: [ '3.9', '3.10' ,'3.11', '3.12']
+          serverVersion: [ '7.0' ]
         fail-fast: false
 
       steps:

README.md

@@ -7,7 +7,7 @@ Install from [PyPi](https://pypi.python.org/pypi), as [ravendb](https://pypi.org
 pip install ravendb
 ````
 ## Introduction and changelog
-Python client API (v5.2) for [RavenDB](https://ravendb.net/) , a NoSQL document database.
+Python client API (v7.0) for [RavenDB](https://ravendb.net/) , a NoSQL document database.
 
 Although new API isn't compatible with the previous one, it comes with **many improvements and new features**.
 
@@ -17,8 +17,6 @@ Although new API isn't compatible with the previous one, it comes with **many im
 
 ## Releases
 
-* All client versions 5.2.x are fully compatible with and support RavenDB server releases 5.4 and 6.0.
-
 * [Click here](https://github.com/ravendb/ravendb-python-client/releases) to view all Releases and Changelog.
 
 ---

README_pypi.md

@@ -8,7 +8,7 @@ pip install ravendb
 ```
 
 ## Introduction
-Python client API (v5.2) for [RavenDB](https://ravendb.net/) , a NoSQL document database.
+Python client API (v7.0) for [RavenDB](https://ravendb.net/) , a NoSQL document database.
 
 Although new API isn't compatible with the previous one, it comes with **many improvements and new features**.
 
@@ -19,8 +19,6 @@ Although new API isn't compatible with the previous one, it comes with **many im
 
 ## Releases
 
-* All client versions 5.2.x are fully compatible with and support RavenDB server releases 5.4 and 6.0.
-
 * [Click here](https://github.com/ravendb/ravendb-python-client/releases) to view all Releases and Changelog.
 
 ## What's new?

pyproject.toml

@@ -1,4 +1,4 @@
 [tool.black]
 line-length = 120
-target-version = ['py37']
+target-version = ['py39']
 include = '\.pyi?$'

ravendb/documents/indexes/abstract_index_creation_tasks.py

@@ -1,5 +1,5 @@
 from abc import abstractmethod, ABC
-from typing import Generic, TypeVar, Union, Dict, Set, Callable, Optional, List, TYPE_CHECKING
+from typing import Generic, TypeVar, Union, Dict, Set, Callable, Optional, List, TYPE_CHECKING, Any
 
 from ravendb.documents.conventions import DocumentConventions
 from ravendb.documents.indexes.definitions import (
@@ -16,8 +16,10 @@
     SpatialOptions,
     IndexFieldOptions,
     IndexType,
+    SearchEngineType,
 )
 from ravendb.documents.indexes.spatial.configuration import SpatialOptionsFactory
+from ravendb.documents.indexes.vector.options import VectorOptions
 from ravendb.documents.operations.indexes import PutIndexesOperation
 from ravendb.documents.store.definition import DocumentStoreBase
 from ravendb.primitives import constants
@@ -40,13 +42,15 @@ def __init__(
         lock_mode: IndexLockMode = None,
         deployment_mode: IndexDeploymentMode = None,
         state: IndexState = None,
+        search_engine_type: SearchEngineType = None,
     ):
         super().__init__()
         self.conventions = conventions
         self.priority = priority
         self.lock_mode = lock_mode
         self.deployment_mode = deployment_mode
         self.state = state
+        self.search_engine_type = search_engine_type
 
     def execute(self, store: "DocumentStore", conventions: DocumentConventions = None, database: str = None):
         old_conventions = self.conventions
@@ -68,6 +72,11 @@ def execute(self, store: "DocumentStore", conventions: DocumentConventions = Non
             if self.deployment_mode is not None:
                 index_definition.deployment_mode = self.deployment_mode
 
+            if self.search_engine_type is not None:
+                if not index_definition.configuration:
+                    index_definition.configuration = {}
+                index_definition.configuration["Indexing.Static.SearchEngineType"] = self.search_engine_type.__str__()
+
             store.maintenance.for_database(database).send(PutIndexesOperation(index_definition))
 
         finally:
@@ -88,6 +97,7 @@ def __init__(self):
         self._index_suggestions: Set[str] = set()
         self._term_vectors_strings: Dict[str, FieldTermVector] = {}
         self._spatial_options_strings: Dict[str, SpatialOptions] = {}
+        self._vector_indexes_strings: Dict[str, VectorOptions] = {}
 
         self._output_reduce_to_collection: Union[None, str] = None
         self._pattern_for_output_reduce_to_collection_references: Union[None, str] = None
@@ -150,6 +160,9 @@ def _add_assembly(self, assembly: AdditionalAssembly) -> None:
 
         self.additional_assemblies.add(assembly)
 
+    def _vector(self, field: str, vector_options: VectorOptions) -> None:
+        self._vector_indexes_strings[field] = vector_options
+
 
 class AbstractIndexDefinitionBuilder(Generic[_T_IndexDefinition]):
     def __init__(self, index_name: str):
@@ -165,6 +178,7 @@ def __init__(self, index_name: str):
         self.suggestions_options: Set[str] = set()
         self.term_vectors_strings: Dict[str, FieldTermVector] = {}
         self.spatial_indexes_strings: Dict[str, SpatialOptions] = {}
+        self.vector_indexes_strings: Dict[str, VectorOptions] = {}
 
         self.lock_mode: Optional[IndexLockMode] = None
         self.priority: Optional[IndexLockMode] = None
@@ -191,7 +205,7 @@ def __apply_values(
         self,
         index_definition: IndexDefinition,
         values: Dict[str, object],
-        action: Callable[[IndexFieldOptions, object], None],
+        action: Callable[[IndexFieldOptions, Any], None],
     ) -> None:
         for key, value in values.items():
             field = index_definition.fields.get(key, IndexFieldOptions())
@@ -216,29 +230,33 @@ def to_index_definition(self, conventions: DocumentConventions, validate_map: bo
             for suggestions_option in self.suggestions_options:
                 suggestions[suggestions_option] = True
 
-            def __set_indexing(options, value):
+            def __set_indexing(options: IndexFieldOptions, value: FieldIndexing):
                 options.indexing = value
 
-            def __set_storage(options, value):
+            def __set_storage(options: IndexFieldOptions, value: FieldStorage):
                 options.storage = value
 
-            def __set_analyzer(options, value):
+            def __set_analyzer(options: IndexFieldOptions, value: str):
                 options.analyzer = value
 
-            def __set_term_vector(options, value):
+            def __set_term_vector(options: IndexFieldOptions, value: FieldTermVector):
                 options.term_vector = value
 
-            def __set_spatial(options, value):
+            def __set_spatial(options: IndexFieldOptions, value: SpatialOptions):
                 options.spatial = value
 
-            def __set_suggestions(options, value):
+            def __set_vector(options: IndexFieldOptions, value: VectorOptions):
+                options.vector = value
+
+            def __set_suggestions(options: IndexFieldOptions, value: bool):
                 options.suggestions = value
 
             self.__apply_values(index_definition, self.indexes_strings, __set_indexing)
             self.__apply_values(index_definition, self.stores_strings, __set_storage)
             self.__apply_values(index_definition, self.analyzers_strings, __set_analyzer)
             self.__apply_values(index_definition, self.term_vectors_strings, __set_term_vector)
             self.__apply_values(index_definition, self.spatial_indexes_strings, __set_spatial)
+            self.__apply_values(index_definition, self.vector_indexes_strings, __set_vector)
             self.__apply_values(index_definition, suggestions, __set_suggestions)
 
             index_definition.additional_sources = self.additional_sources
@@ -302,6 +320,7 @@ def create_index_definition(self) -> IndexDefinition:
         index_definition_builder.suggestions_options = self._index_suggestions
         index_definition_builder.term_vectors_strings = self._term_vectors_strings
         index_definition_builder.spatial_indexes_strings = self._spatial_options_strings
+        index_definition_builder.vector_indexes_strings = self._vector_indexes_strings
         index_definition_builder.output_reduce_to_collection = self._output_reduce_to_collection
         index_definition_builder.pattern_for_output_reduce_to_collection_references = (
             self._pattern_for_output_reduce_to_collection_references

ravendb/documents/indexes/definitions.py

@@ -6,6 +6,7 @@
 from abc import ABC
 from typing import Union, Optional, List, Dict, Set, Iterable
 from ravendb.documents.indexes.spatial.configuration import SpatialOptions, AutoSpatialOptions
+from ravendb.documents.indexes.vector.options import VectorOptions, AutoVectorOptions
 from ravendb.tools.utils import Utils
 
 
@@ -60,6 +61,14 @@ def __str__(self):
         return self.value
 
 
+class SearchEngineType(Enum):
+    LUCENE = "Lucene"
+    CORAX = "Corax"
+
+    def __str__(self):
+        return self.value
+
+
 class FieldStorage(Enum):
     YES = "Yes"
     NO = "No"
@@ -144,13 +153,15 @@ def __init__(
         indexing: Optional[FieldIndexing] = None,
         term_vector: Optional[FieldTermVector] = None,
         spatial: Optional[SpatialOptions] = None,
+        vector: Optional[VectorOptions] = None,
         analyzer: Optional[str] = None,
         suggestions: Optional[bool] = None,
     ):
         self.storage = storage
         self.indexing = indexing
         self.term_vector = term_vector
         self.spatial = spatial
+        self.vector = vector
         self.analyzer = analyzer
         self.suggestions = suggestions
 
@@ -160,6 +171,7 @@ def to_json(self):
             "Indexing": self.indexing,
             "TermVector": self.term_vector,
             "Spatial": self.spatial.to_json() if self.spatial else None,
+            "Vector": self.vector.to_json() if self.vector else None,
             "Analyzer": self.analyzer,
             "Suggestions": self.suggestions,
         }
@@ -192,6 +204,7 @@ def __init__(
         pattern_for_output_reduce_to_collection_references: Optional[str] = None,
         pattern_references_collection_name: Optional[str] = None,
         deployment_mode: Optional[IndexDeploymentMode] = None,
+        search_engine_type: Optional[SearchEngineType] = None,
     ):
         super(IndexDefinition, self).__init__(name, priority, state)
         self.lock_mode = lock_mode
@@ -208,6 +221,7 @@ def __init__(
         self.pattern_for_output_reduce_to_collection_references = pattern_for_output_reduce_to_collection_references
         self.pattern_references_collection_name = pattern_references_collection_name
         self.deployment_mode = deployment_mode
+        self.search_engine_type = search_engine_type
 
     @classmethod
     def from_json(cls, json_dict: dict) -> IndexDefinition:
@@ -230,6 +244,8 @@ def from_json(cls, json_dict: dict) -> IndexDefinition:
         index_type = json_dict.get("IndexType", None)
         if index_type is not None:
             result.__index_type = IndexType(index_type)
+        if json_dict["Configuration"] and "Indexing.Static.SearchEngineType" in json_dict["Configuration"]:
+            result.search_engine_type = SearchEngineType(json_dict["Configuration"]["Indexing.Static.SearchEngineType"])
         result.output_reduce_to_collection = json_dict["OutputReduceToCollection"]
         result.reduce_output_index = json_dict["ReduceOutputIndex"]
         result.pattern_for_output_reduce_to_collection_references = json_dict[
@@ -370,6 +386,7 @@ def __init__(
         indexing: Optional[AutoFieldIndexing] = None,
         aggregation: Optional[AggregationOperation] = None,
         spatial: Optional[AutoSpatialOptions] = None,
+        vector: Optional[AutoVectorOptions] = None,
         group_by_array_behavior: Optional[GroupByArrayBehavior] = None,
         suggestions: Optional[bool] = None,
         is_name_quoted: Optional[bool] = None,
@@ -378,6 +395,7 @@ def __init__(
         self.indexing = indexing
         self.aggregation = aggregation
         self.spatial = spatial
+        self.vector = vector
         self.group_by_array_behavior = group_by_array_behavior
         self.suggestions = suggestions
         self.is_name_quoted = is_name_quoted
@@ -389,6 +407,7 @@ def from_json(cls, json_dict: Dict) -> AutoIndexFieldOptions:
             AutoFieldIndexing(json_dict.get("Indexing")),
             AggregationOperation(json_dict.get("Aggregation")) if json_dict.get("Aggregation", None) else None,
             AutoSpatialOptions.from_json(json_dict.get("Spatial")) if json_dict.get("Spatial", None) else None,
+            AutoVectorOptions.from_json(json_dict.get("Vector")) if json_dict.get("Vector", None) else None,
             GroupByArrayBehavior(json_dict.get("GroupByArrayBehavior")),
             json_dict.get("Suggestions"),
             json_dict.get("IsNameQuoted"),
@@ -400,6 +419,9 @@ def to_json(self) -> Dict:
             "Indexing": self.indexing.value,
             "Aggregation": self.aggregation.value if self.aggregation is not None else None,
             "Spatial": self.spatial.type if self.spatial is not None else None,
+            "Vector": (
+                self.vector.to_json() if self.vector is not None else None
+            ),  # todo; check if vector.to_json() is valid here
             "GroupByArrayBehavior": self.group_by_array_behavior.value,
             "Suggestions": self.suggestions,
             "IsNameQuoted": self.is_name_quoted,

ravendb/documents/indexes/vector/__init__.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class VectorEmbeddingType(Enum):
+    SINGLE = "Single"  # float
+    INT8 = "Int8"  # quantized int
+    BINARY = "Binary"  # 1/0 quantized int
+    TEXT = "Text"  # str

ravendb/documents/indexes/vector/embedding.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from typing import Dict, Any
+
+from ravendb.documents.indexes.vector.embedding import VectorEmbeddingType
+from ravendb.primitives import constants
+
+
+class VectorOptions:
+    def __init__(
+        self,
+        source_embedding_type: VectorEmbeddingType = constants.VectorSearch.DEFAULT_EMBEDDING_TYPE,
+        destination_embedding_type: VectorEmbeddingType = constants.VectorSearch.DEFAULT_EMBEDDING_TYPE,
+        dimensions: int = None,
+        number_of_edges: int = None,
+        number_of_candidates_for_indexing: int = None,
+    ):
+        self.dimensions = dimensions
+        self.source_embedding_type = source_embedding_type
+        self.destination_embedding_type = destination_embedding_type
+        self.numbers_of_candidates_for_indexing = number_of_candidates_for_indexing
+        self.number_of_edges = number_of_edges
+
+    @classmethod
+    def from_json(cls, json_dict: Dict[str, Any]) -> VectorOptions:
+        return cls(
+            json_dict["SourceEmbeddingType"],
+            json_dict["DestinationEmbeddingType"],
+            json_dict["Dimensions"],
+            json_dict["NumberOfEdges"],
+            json_dict["NumberOfCandidatesForIndexing"],
+        )
+
+    def to_json(self) -> Dict[str, Any]:
+        return {
+            "SourceEmbeddingType": self.source_embedding_type.value,
+            "DestinationEmbeddingType": self.destination_embedding_type.value,
+            "Dimensions": self.dimensions,
+            "NumberOfCandidatesForIndexing": self.numbers_of_candidates_for_indexing,
+            "NumberOfEdges": self.number_of_edges,
+        }
+
+
+class AutoVectorOptions(VectorOptions):
+    def __init__(
+        self,
+        source_embedding_type: VectorEmbeddingType = constants.VectorSearch.DEFAULT_EMBEDDING_TYPE,
+        destination_embedding_type: VectorEmbeddingType = constants.VectorSearch.DEFAULT_EMBEDDING_TYPE,
+        dimensions: int = None,
+        number_of_edges: int = None,
+        number_of_candidates_for_indexing: int = None,
+        source_field_name: str = None,
+    ):
+        super().__init__(
+            source_embedding_type,
+            destination_embedding_type,
+            dimensions,
+            number_of_edges,
+            number_of_candidates_for_indexing,
+        )
+        self.source_field_name = source_field_name
+
+    @classmethod
+    def from_vector_options(cls, vector_options: VectorOptions):
+        return cls(
+            source_embedding_type=vector_options.source_embedding_type,
+            destination_embedding_type=vector_options.destination_embedding_type,
+            dimensions=vector_options.dimensions,
+            number_of_edges=vector_options.number_of_edges,
+            number_of_candidates_for_indexing=vector_options.numbers_of_candidates_for_indexing,
+        )
+
+    @classmethod
+    def from_json(cls, json_dict: Dict[str, Any]):
+        vec_options = super().from_json(json_dict)
+        auto_vect_options = cls.from_vector_options(vec_options)
+        auto_vect_options.source_field_name = json_dict["SourceFieldName"]
+        return auto_vect_options
+
+    def to_json(self) -> Dict[str, Any]:
+        json_dict = super().to_json()
+        json_dict["SourceFieldName"] = self.source_field_name
+        return json_dict