Add grpc endpoint for Facet (#4933)

* add grpc endpoint * gen grpc docs
qdrant · Aug 22, 2024 · b92908f · b92908f
1 parent 06e1e33
commit b92908f
Show file tree

Hide file tree

Showing 12 changed files with 293 additions and 36 deletions.
diff --git a/docs/grpc/docs.md b/docs/grpc/docs.md
@@ -138,8 +138,10 @@
     - [DiscoverInput](#qdrant-DiscoverInput)
     - [DiscoverPoints](#qdrant-DiscoverPoints)
     - [DiscoverResponse](#qdrant-DiscoverResponse)
+    - [FacetCounts](#qdrant-FacetCounts)
+    - [FacetHit](#qdrant-FacetHit)
+    - [FacetResponse](#qdrant-FacetResponse)
     - [FacetValue](#qdrant-FacetValue)
-    - [FacetValueHit](#qdrant-FacetValueHit)
     - [FieldCondition](#qdrant-FieldCondition)
     - [Filter](#qdrant-Filter)
     - [GeoBoundingBox](#qdrant-GeoBoundingBox)
@@ -2411,24 +2413,31 @@ The JSON representation for `Value` is a JSON value.
 
 
 
-<a name="qdrant-FacetValue"></a>
+<a name="qdrant-FacetCounts"></a>
 
-### FacetValue
+### FacetCounts
 
 
 
 | Field | Type | Label | Description |
 | ----- | ---- | ----- | ----------- |
-| string_value | [string](#string) |  | String value from the facet |
+| collection_name | [string](#string) |  | Name of the collection |
+| key | [string](#string) |  | Payload key of the facet |
+| filter | [Filter](#qdrant-Filter) | optional | Filter conditions - return only those points that satisfy the specified conditions. |
+| limit | [uint64](#uint64) | optional | Max number of facets. Default is 10. |
+| exact | [bool](#bool) | optional | If true, return exact counts, slower but useful for debugging purposes. Default is false. |
+| timeout | [uint64](#uint64) | optional | If set, overrides global timeout setting for this request. Unit is seconds. |
+| read_consistency | [ReadConsistency](#qdrant-ReadConsistency) | optional | Options for specifying read consistency guarantees |
+| shard_key_selector | [ShardKeySelector](#qdrant-ShardKeySelector) | optional | Specify in which shards to look for the points, if not specified - look in all shards |
 
 
 
 
 
 
-<a name="qdrant-FacetValueHit"></a>
+<a name="qdrant-FacetHit"></a>
 
-### FacetValueHit
+### FacetHit
 
 
 
@@ -2442,6 +2451,37 @@ The JSON representation for `Value` is a JSON value.
 
 
 
+<a name="qdrant-FacetResponse"></a>
+
+### FacetResponse
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| hits | [FacetHit](#qdrant-FacetHit) | repeated |  |
+| time | [double](#double) |  | Time spent to process |
+
+
+
+
+
+
+<a name="qdrant-FacetValue"></a>
+
+### FacetValue
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| string_value | [string](#string) |  | String value from the facet |
+
+
+
+
+
+
 <a name="qdrant-FieldCondition"></a>
 
 ### FieldCondition
@@ -4371,6 +4411,7 @@ When using target (with or without context), the score behaves a little differen
 | Query | [QueryPoints](#qdrant-QueryPoints) | [QueryResponse](#qdrant-QueryResponse) | Universally query points. This endpoint covers all capabilities of search, recommend, discover, filters. But also enables hybrid and multi-stage queries. |
 | QueryBatch | [QueryBatchPoints](#qdrant-QueryBatchPoints) | [QueryBatchResponse](#qdrant-QueryBatchResponse) | Universally query points in a batch fashion. This endpoint covers all capabilities of search, recommend, discover, filters. But also enables hybrid and multi-stage queries. |
 | QueryGroups | [QueryPointGroups](#qdrant-QueryPointGroups) | [QueryGroupsResponse](#qdrant-QueryGroupsResponse) | Universally query points in a group fashion. This endpoint covers all capabilities of search, recommend, discover, filters. But also enables hybrid and multi-stage queries. |
+| Facet | [FacetCounts](#qdrant-FacetCounts) | [FacetResponse](#qdrant-FacetResponse) | Perform facet counts. For each value in the field, count the number of points that have this value and match the conditions. |
 
 
 

diff --git a/lib/api/build.rs b/lib/api/build.rs
@@ -270,6 +270,9 @@ fn configure_validation(builder: Builder) -> Builder {
             ("QueryBatchPoints.collection_name", "length(min = 1, max = 255)"),
             ("QueryBatchPoints.query_points", ""),
             ("QueryBatchPoints.timeout", "custom(function = \"crate::grpc::validate::validate_u64_range_min_1\")"),
+            ("FacetCounts.collection_name", "length(min = 1, max = 255)"),
+            ("FacetCounts.key", "length(min = 1)"),
+            ("FacetCounts.timeout", "custom(function = \"crate::grpc::validate::validate_u64_range_min_1\")")
         ], &[])
         .type_attribute(".", "#[derive(serde::Serialize)]")
         // Service: points_internal_service.proto

diff --git a/lib/api/src/grpc/conversions.rs b/lib/api/src/grpc/conversions.rs
@@ -19,7 +19,7 @@ use uuid::Uuid;
 use super::qdrant::raw_query::RawContextPair;
 use super::qdrant::{
     raw_query, start_from, BinaryQuantization, BoolIndexParams, CompressionRatio,
-    DatetimeIndexParams, DatetimeRange, Direction, FacetValue, FacetValueHit, FieldType,
+    DatetimeIndexParams, DatetimeRange, Direction, FacetHit, FacetValue, FieldType,
     FloatIndexParams, GeoIndexParams, GeoLineString, GroupId, KeywordIndexParams, LookupLocation,
     MultiVectorComparator, MultiVectorConfig, OrderBy, OrderValue, Range, RawVector,
     RecommendStrategy, SearchPointGroups, SearchPoints, ShardKeySelector, SparseIndices, StartFrom,
@@ -2236,13 +2236,13 @@ impl From<LookupLocation> for rest::LookupLocation {
     }
 }
 
-impl TryFrom<FacetValueHit> for segment_facets::FacetValueHit {
+impl TryFrom<FacetHit> for segment_facets::FacetValueHit {
     type Error = Status;
 
-    fn try_from(hit: FacetValueHit) -> Result<Self, Self::Error> {
+    fn try_from(hit: FacetHit) -> Result<Self, Self::Error> {
         let value = hit
             .value
-            .ok_or_else(|| Status::internal("expected FacetValueHit to have a value"))?;
+            .ok_or_else(|| Status::internal("expected FacetHit to have a value"))?;
 
         Ok(Self {
             value: segment_facets::FacetValue::try_from(value)?,
@@ -2251,7 +2251,7 @@ impl TryFrom<FacetValueHit> for segment_facets::FacetValueHit {
     }
 }
 
-impl From<segment_facets::FacetValueHit> for FacetValueHit {
+impl From<segment_facets::FacetValueHit> for FacetHit {
     fn from(hit: segment_facets::FacetValueHit) -> Self {
         Self {
             value: Some(hit.value.into()),

diff --git a/lib/api/src/grpc/proto/points.proto b/lib/api/src/grpc/proto/points.proto
@@ -589,13 +589,24 @@ message QueryPointGroups {
   optional ShardKeySelector shard_key_selector = 17; // Specify in which shards to look for the points, if not specified - look in all shards
 }
 
+message FacetCounts {
+    string collection_name = 1; // Name of the collection
+    string key = 2; // Payload key of the facet
+    optional Filter filter = 3; // Filter conditions - return only those points that satisfy the specified conditions.
+    optional uint64 limit = 4; // Max number of facets. Default is 10.
+    optional bool exact = 5; // If true, return exact counts, slower but useful for debugging purposes. Default is false.
+    optional uint64 timeout = 6; // If set, overrides global timeout setting for this request. Unit is seconds.
+    optional ReadConsistency read_consistency = 7; // Options for specifying read consistency guarantees
+    optional ShardKeySelector shard_key_selector = 8; // Specify in which shards to look for the points, if not specified - look in all shards
+}
+
 message FacetValue {
     oneof variant {
         string string_value = 1; // String value from the facet
     }
 }
 
-message FacetValueHit {
+message FacetHit {
     FacetValue value = 1; // Value from the facet
     uint64 count = 2; // Number of points with this value
 }
@@ -814,6 +825,11 @@ message UpdateBatchResponse {
   double time = 2; // Time spent to process
 }
 
+message FacetResponse {
+    repeated FacetHit hits = 1;
+    double time = 2; // Time spent to process
+}
+
 // ---------------------------------------------
 // ------------- Filter Conditions -------------
 // ---------------------------------------------

diff --git a/lib/api/src/grpc/proto/points_internal_service.proto b/lib/api/src/grpc/proto/points_internal_service.proto
@@ -300,6 +300,6 @@ message FacetCountsInternal {
 }
 
 message FacetResponseInternal {
-    repeated FacetValueHit hits = 1;
+    repeated FacetHit hits = 1;
     double time = 2; // Time spent to process
 }
diff --git a/lib/api/src/grpc/proto/points_service.proto b/lib/api/src/grpc/proto/points_service.proto
@@ -121,4 +121,8 @@ service Points {
   Universally query points in a group fashion. This endpoint covers all capabilities of search, recommend, discover, filters. But also enables hybrid and multi-stage queries.
   */
   rpc QueryGroups (QueryPointGroups) returns (QueryGroupsResponse) {}
+  /*
+  Perform facet counts. For each value in the field, count the number of points that have this value and match the conditions.
+  */
+  rpc Facet (FacetCounts) returns (FacetResponse) {}
 }
diff --git a/lib/api/src/grpc/qdrant.rs b/lib/api/src/grpc/qdrant.rs
@@ -5028,6 +5028,39 @@ pub struct QueryPointGroups {
     #[prost(message, optional, tag = "17")]
     pub shard_key_selector: ::core::option::Option<ShardKeySelector>,
 }
+#[derive(validator::Validate)]
+#[derive(serde::Serialize)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct FacetCounts {
+    /// Name of the collection
+    #[prost(string, tag = "1")]
+    #[validate(length(min = 1, max = 255))]
+    pub collection_name: ::prost::alloc::string::String,
+    /// Payload key of the facet
+    #[prost(string, tag = "2")]
+    #[validate(length(min = 1))]
+    pub key: ::prost::alloc::string::String,
+    /// Filter conditions - return only those points that satisfy the specified conditions.
+    #[prost(message, optional, tag = "3")]
+    pub filter: ::core::option::Option<Filter>,
+    /// Max number of facets. Default is 10.
+    #[prost(uint64, optional, tag = "4")]
+    pub limit: ::core::option::Option<u64>,
+    /// If true, return exact counts, slower but useful for debugging purposes. Default is false.
+    #[prost(bool, optional, tag = "5")]
+    pub exact: ::core::option::Option<bool>,
+    /// If set, overrides global timeout setting for this request. Unit is seconds.
+    #[prost(uint64, optional, tag = "6")]
+    #[validate(custom(function = "crate::grpc::validate::validate_u64_range_min_1"))]
+    pub timeout: ::core::option::Option<u64>,
+    /// Options for specifying read consistency guarantees
+    #[prost(message, optional, tag = "7")]
+    pub read_consistency: ::core::option::Option<ReadConsistency>,
+    /// Specify in which shards to look for the points, if not specified - look in all shards
+    #[prost(message, optional, tag = "8")]
+    pub shard_key_selector: ::core::option::Option<ShardKeySelector>,
+}
 #[derive(serde::Serialize)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -5049,7 +5082,7 @@ pub mod facet_value {
 #[derive(serde::Serialize)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
-pub struct FacetValueHit {
+pub struct FacetHit {
     /// Value from the facet
     #[prost(message, optional, tag = "1")]
     pub value: ::core::option::Option<FacetValue>,
@@ -5519,6 +5552,16 @@ pub struct UpdateBatchResponse {
     #[prost(double, tag = "2")]
     pub time: f64,
 }
+#[derive(serde::Serialize)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct FacetResponse {
+    #[prost(message, repeated, tag = "1")]
+    pub hits: ::prost::alloc::vec::Vec<FacetHit>,
+    /// Time spent to process
+    #[prost(double, tag = "2")]
+    pub time: f64,
+}
 #[derive(validator::Validate)]
 #[derive(serde::Serialize)]
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -6800,6 +6843,26 @@ pub mod points_client {
             req.extensions_mut().insert(GrpcMethod::new("qdrant.Points", "QueryGroups"));
             self.inner.unary(req, path, codec).await
         }
+        /// Perform facet counts. For each value in the field, count the number of points that have this value and match the conditions.
+        pub async fn facet(
+            &mut self,
+            request: impl tonic::IntoRequest<super::FacetCounts>,
+        ) -> std::result::Result<tonic::Response<super::FacetResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/qdrant.Points/Facet");
+            let mut req = request.into_request();
+            req.extensions_mut().insert(GrpcMethod::new("qdrant.Points", "Facet"));
+            self.inner.unary(req, path, codec).await
+        }
     }
 }
 /// Generated server implementations.
@@ -7008,6 +7071,11 @@ pub mod points_server {
             tonic::Response<super::QueryGroupsResponse>,
             tonic::Status,
         >;
+        /// Perform facet counts. For each value in the field, count the number of points that have this value and match the conditions.
+        async fn facet(
+            &self,
+            request: tonic::Request<super::FacetCounts>,
+        ) -> std::result::Result<tonic::Response<super::FacetResponse>, tonic::Status>;
     }
     #[derive(Debug)]
     pub struct PointsServer<T: Points> {
@@ -8206,6 +8274,50 @@ pub mod points_server {
                     };
                     Box::pin(fut)
                 }
+                "/qdrant.Points/Facet" => {
+                    #[allow(non_camel_case_types)]
+                    struct FacetSvc<T: Points>(pub Arc<T>);
+                    impl<T: Points> tonic::server::UnaryService<super::FacetCounts>
+                    for FacetSvc<T> {
+                        type Response = super::FacetResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::FacetCounts>,
+                        ) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move {
+                                <T as Points>::facet(&inner, request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = FacetSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            )
+                            .apply_max_message_size_config(
+                                max_decoding_message_size,
+                                max_encoding_message_size,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 _ => {
                     Box::pin(async move {
                         Ok(
@@ -8861,7 +8973,7 @@ pub struct FacetCountsInternal {
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct FacetResponseInternal {
     #[prost(message, repeated, tag = "1")]
-    pub hits: ::prost::alloc::vec::Vec<FacetValueHit>,
+    pub hits: ::prost::alloc::vec::Vec<FacetHit>,
     /// Time spent to process
     #[prost(double, tag = "2")]
     pub time: f64,

diff --git a/src/common/metrics.rs b/src/common/metrics.rs
@@ -57,7 +57,7 @@ const GRPC_ENDPOINT_WHITELIST: &[&str] = &[
     "/qdrant.Points/DeletePayload",
     "/qdrant.Points/Discover",
     "/qdrant.Points/DiscoverBatch",
-    // TODO(facet): add GRPC endpoint for facets here
+    "/qdrant.Points/Facet",
     "/qdrant.Points/Get",
     "/qdrant.Points/OverwritePayload",
     "/qdrant.Points/Query",