Add strict mode config to update collection API + Populate strict mod…

…e in API (qdrant#5187) * add strict mode to update collection API * update API specs * clippy * Update lib/storage/src/content_manager/toc/collection_meta_ops.rs * Strict mode integration tests (qdrant#5189) * Add integration tests for strict mode * add full update test * Apply suggestions from code review Co-authored-by: Tim Visée <tim+github@visee.me> * Adjust error messages * improve checking of error response --------- Co-authored-by: Tim Visée <tim+github@visee.me> * prefer explicit structu conversion --------- Co-authored-by: Tim Visée <tim+github@visee.me> Co-authored-by: generall <andrey@vasnetsov.com>
AKJUS · Nov 8, 2024 · a0de950 · a0de950
1 parent eaa338d
commit a0de950
Show file tree

Hide file tree

Showing 10 changed files with 490 additions and 8 deletions.
diff --git a/docs/grpc/docs.md b/docs/grpc/docs.md
@@ -1469,6 +1469,7 @@ Note: 1kB = 1 vector of size 256. |
 | vectors_config | [VectorsConfigDiff](#qdrant-VectorsConfigDiff) | optional | New vector parameters |
 | quantization_config | [QuantizationConfigDiff](#qdrant-QuantizationConfigDiff) | optional | Quantization configuration of vector |
 | sparse_vectors_config | [SparseVectorConfig](#qdrant-SparseVectorConfig) | optional | New sparse vector parameters |
+| strict_mode_config | [StrictModeConfig](#qdrant-StrictModeConfig) | optional | New strict mode configuration |
 
 
 

diff --git a/docs/redoc/master/openapi.json b/docs/redoc/master/openapi.json
@@ -8683,6 +8683,17 @@
               "$ref": "#/components/schemas/SparseVectorParams"
             },
             "nullable": true
+          },
+          "strict_mode_config": {
+            "description": "Strict-mode config.",
+            "anyOf": [
+              {
+                "$ref": "#/components/schemas/StrictModeConfig"
+              },
+              {
+                "nullable": true
+              }
+            ]
           }
         }
       },
@@ -8777,6 +8788,58 @@
           }
         }
       },
+      "StrictModeConfig": {
+        "type": "object",
+        "properties": {
+          "enabled": {
+            "description": "Whether strict mode is enabled for a collection or not.",
+            "type": "boolean",
+            "nullable": true
+          },
+          "max_query_limit": {
+            "description": "Max allowed `limit` parameter for all APIs that don't have their own max limit.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 1,
+            "nullable": true
+          },
+          "max_timeout": {
+            "description": "Max allowed `timeout` parameter.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 1,
+            "nullable": true
+          },
+          "unindexed_filtering_retrieve": {
+            "description": "Allow usage of unindexed fields in retrieval based (eg. search) filters.",
+            "type": "boolean",
+            "nullable": true
+          },
+          "unindexed_filtering_update": {
+            "description": "Allow usage of unindexed fields in filtered updates (eg. delete by payload).",
+            "type": "boolean",
+            "nullable": true
+          },
+          "search_max_hnsw_ef": {
+            "description": "Max HNSW value allowed in search parameters.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0,
+            "nullable": true
+          },
+          "search_allow_exact": {
+            "description": "Whether exact search is allowed or not.",
+            "type": "boolean",
+            "nullable": true
+          },
+          "search_max_oversampling": {
+            "description": "Max oversampling value allowed in search.",
+            "type": "number",
+            "format": "double",
+            "nullable": true
+          }
+        }
+      },
       "UpdateCollection": {
         "description": "Operation for updating parameters of the existing collection",
         "type": "object",
@@ -8847,6 +8910,16 @@
                 "nullable": true
               }
             ]
+          },
+          "strict_mode_config": {
+            "anyOf": [
+              {
+                "$ref": "#/components/schemas/StrictModeConfig"
+              },
+              {
+                "nullable": true
+              }
+            ]
           }
         }
       },

diff --git a/lib/api/src/grpc/proto/collections.proto b/lib/api/src/grpc/proto/collections.proto
@@ -351,6 +351,7 @@ message UpdateCollection {
   optional VectorsConfigDiff vectors_config = 6; // New vector parameters
   optional QuantizationConfigDiff quantization_config = 7; // Quantization configuration of vector
   optional SparseVectorConfig sparse_vectors_config = 8; // New sparse vector parameters
+  optional StrictModeConfig strict_mode_config = 9; // New strict mode configuration
 }
 
 message DeleteCollection {

diff --git a/lib/api/src/grpc/qdrant.rs b/lib/api/src/grpc/qdrant.rs
@@ -549,6 +549,9 @@ pub struct UpdateCollection {
     /// New sparse vector parameters
     #[prost(message, optional, tag = "8")]
     pub sparse_vectors_config: ::core::option::Option<SparseVectorConfig>,
+    /// New strict mode configuration
+    #[prost(message, optional, tag = "9")]
+    pub strict_mode_config: ::core::option::Option<StrictModeConfig>,
 }
 #[derive(validator::Validate)]
 #[derive(serde::Serialize)]

diff --git a/lib/collection/src/collection/collection_ops.rs b/lib/collection/src/collection/collection_ops.rs
@@ -162,6 +162,23 @@ impl Collection {
         Ok(())
     }
 
+    /// Updates the strict mode configuration and saves it to disk.
+    pub async fn update_strict_mode_config(
+        &self,
+        strict_mode_diff: StrictModeConfig,
+    ) -> CollectionResult<()> {
+        {
+            let mut config = self.collection_config.write().await;
+            if let Some(current_config) = config.strict_mode_config.as_mut() {
+                *current_config = strict_mode_diff.update(current_config)?;
+            } else {
+                config.strict_mode_config = Some(strict_mode_diff);
+            }
+        }
+        self.collection_config.read().await.save(&self.path)?;
+        Ok(())
+    }
+
     /// Handle replica changes
     ///
     /// add and remove replicas from replica set

diff --git a/lib/storage/src/content_manager/collection_meta_ops.rs b/lib/storage/src/content_manager/collection_meta_ops.rs
@@ -170,7 +170,6 @@ pub struct CreateCollection {
     pub sparse_vectors: Option<BTreeMap<String, SparseVectorParams>>,
     /// Strict-mode config.
     #[validate(nested)]
-    #[schemars(skip)]
     pub strict_mode_config: Option<StrictModeConfig>,
 }
 
@@ -229,6 +228,8 @@ pub struct UpdateCollection {
     /// Map of sparse vector data parameters to update for each sparse vector.
     #[validate(nested)]
     pub sparse_vectors: Option<SparseVectorsConfig>,
+    #[validate(nested)]
+    pub strict_mode_config: Option<StrictModeConfig>,
 }
 
 /// Operation for updating parameters of the existing collection
@@ -251,6 +252,7 @@ impl UpdateCollectionOperation {
                 optimizers_config: None,
                 quantization_config: None,
                 sparse_vectors: None,
+                strict_mode_config: None,
             },
             shard_replica_changes: None,
         }

diff --git a/lib/storage/src/content_manager/conversions.rs b/lib/storage/src/content_manager/conversions.rs
@@ -1,5 +1,8 @@
+use collection::operations::config_diff::{
+    CollectionParamsDiff, HnswConfigDiff, OptimizersConfigDiff, QuantizationConfigDiff,
+};
 use collection::operations::conversions::sharding_method_from_proto;
-use collection::operations::types::SparseVectorsConfig;
+use collection::operations::types::{SparseVectorsConfig, VectorsConfigDiff};
 use segment::types::StrictModeConfig;
 use tonic::Status;
 
@@ -92,19 +95,23 @@ impl TryFrom<api::grpc::qdrant::UpdateCollection> for CollectionMetaOperations {
                 vectors: value
                     .vectors_config
                     .and_then(|config| config.config)
-                    .map(TryInto::try_into)
+                    .map(VectorsConfigDiff::try_from)
+                    .transpose()?,
+                hnsw_config: value.hnsw_config.map(HnswConfigDiff::from),
+                params: value
+                    .params
+                    .map(CollectionParamsDiff::try_from)
                     .transpose()?,
-                hnsw_config: value.hnsw_config.map(Into::into),
-                params: value.params.map(TryInto::try_into).transpose()?,
-                optimizers_config: value.optimizers_config.map(Into::into),
+                optimizers_config: value.optimizers_config.map(OptimizersConfigDiff::from),
                 quantization_config: value
                     .quantization_config
-                    .map(TryInto::try_into)
+                    .map(QuantizationConfigDiff::try_from)
                     .transpose()?,
                 sparse_vectors: value
                     .sparse_vectors_config
-                    .map(TryInto::try_into)
+                    .map(SparseVectorsConfig::try_from)
                     .transpose()?,
+                strict_mode_config: value.strict_mode_config.map(StrictModeConfig::from),
             },
         )))
     }

diff --git a/lib/storage/src/content_manager/mod.rs b/lib/storage/src/content_manager/mod.rs
@@ -135,6 +135,7 @@ pub mod consensus_ops {
                     hnsw_config: None,
                     quantization_config: None,
                     sparse_vectors: None,
+                    strict_mode_config: None,
                 },
             );
             operation

diff --git a/lib/storage/src/content_manager/toc/collection_meta_ops.rs b/lib/storage/src/content_manager/toc/collection_meta_ops.rs
@@ -126,6 +126,7 @@ impl TableOfContent {
             optimizers_config,
             quantization_config,
             sparse_vectors,
+            strict_mode_config: strict_mode,
         } = operation.update_collection;
         let collection = self
             .get_collection_unchecked(&operation.collection_name)
@@ -161,6 +162,9 @@ impl TableOfContent {
         if let Some(changes) = replica_changes {
             collection.handle_replica_changes(changes).await?;
         }
+        if let Some(strict_mode) = strict_mode {
+            collection.update_strict_mode_config(strict_mode).await?;
+        }
 
         // Recreate optimizers
         if recreate_optimizers {