Introduce Cargo feature "testing" (qdrant#4192)

AKJUS · May 9, 2024 · 8fe5e43 · 8fe5e43
1 parent 6ed0011
commit 8fe5e43
Show file tree

Hide file tree

Showing 33 changed files with 59 additions and 39 deletions.
diff --git a/.github/workflows/rust-lint.yml b/.github/workflows/rust-lint.yml
@@ -36,4 +36,8 @@ jobs:
     - name: Check code formatting
       run: cargo +nightly fmt --all -- --check
     - name: Check cargo clippy warnings
+      run: cargo clippy --workspace -- -D warnings
+    - name: Check cargo clippy warnings for all targets
+      run: cargo clippy --workspace --all-targets -- -D warnings
+    - name: Check cargo clippy warnings for all targets and features
       run: cargo clippy --workspace --all-targets --all-features -- -D warnings
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/lib/collection/Cargo.toml b/lib/collection/Cargo.toml
@@ -9,13 +9,16 @@ license = "Apache-2.0"
 edition = "2021"
 
 [features]
+testing = []
 tracing = ["dep:tracing", "api/tracing", "segment/tracing"]
 
 [dev-dependencies]
 criterion = "0.5"
 proptest = "1.4.0"
 rstest = "0.19.0"
 approx = "0.5.1"
+collection = { path = ".", features = ["testing"] }
+common = { path = "../common/common", features = ["testing"] }
 
 [target.'cfg(not(target_os = "windows"))'.dev-dependencies]
 pprof = { version = "0.12", features = ["flamegraph", "prost-codec"] }

diff --git a/lib/collection/src/collection_manager/holders/proxy_segment.rs b/lib/collection/src/collection_manager/holders/proxy_segment.rs
@@ -265,6 +265,7 @@ impl ProxySegment {
     }
 
     /// This function is a simplified version of `search_batch` intended for testing purposes.
+    #[cfg(feature = "testing")]
     #[allow(clippy::too_many_arguments)]
     pub fn search(
         &self,

diff --git a/lib/collection/tests/integration/collection_restore_test.rs b/lib/collection/tests/integration/collection_restore_test.rs
@@ -6,11 +6,12 @@ use collection::operations::types::ScrollRequestInternal;
 use collection::operations::CollectionUpdateOperations;
 use itertools::Itertools;
 use segment::data_types::vectors::BatchVectorStruct;
+use segment::json_path::path;
 use segment::types::{PayloadContainer, PayloadSelectorExclude, WithPayloadInterface};
 use serde_json::Value;
 use tempfile::Builder;
 
-use crate::common::{load_local_collection, path, simple_collection_fixture, N_SHARDS};
+use crate::common::{load_local_collection, simple_collection_fixture, N_SHARDS};
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_collection_reloading() {

diff --git a/lib/collection/tests/integration/common/mod.rs b/lib/collection/tests/integration/common/mod.rs
@@ -81,11 +81,6 @@ pub fn dummy_abort_shard_transfer() -> AbortShardTransfer {
     Arc::new(|_transfer, _reason| {})
 }
 
-#[cfg(test)]
-pub fn path(p: &str) -> segment::json_path::JsonPath {
-    p.parse().unwrap()
-}
-
 /// Default to a collection with all the shards local
 #[cfg(test)]
 pub async fn new_local_collection(

diff --git a/lib/collection/tests/integration/grouping_test.rs b/lib/collection/tests/integration/grouping_test.rs
@@ -10,10 +10,11 @@ use rand::distributions::Uniform;
 use rand::rngs::ThreadRng;
 use rand::Rng;
 use segment::data_types::vectors::DenseVector;
+use segment::json_path::path;
 use segment::types::{Filter, Payload, WithPayloadInterface, WithVector};
 use serde_json::json;
 
-use crate::common::{path, simple_collection_fixture};
+use crate::common::simple_collection_fixture;
 
 fn rand_dense_vector(rng: &mut ThreadRng, size: usize) -> DenseVector {
     rng.sample_iter(Uniform::new(0.4, 0.6)).take(size).collect()

diff --git a/lib/common/common/Cargo.toml b/lib/common/common/Cargo.toml
@@ -9,6 +9,9 @@ license = "Apache-2.0"
 edition = "2021"
 publish = false
 
+[features]
+testing = []
+
 [dependencies]
 num_cpus = "1.16"
 ordered-float = "4.2"

diff --git a/lib/common/common/src/cpu.rs b/lib/common/common/src/cpu.rs
@@ -177,6 +177,7 @@ impl CpuPermit {
     }
 
     /// New CPU permit with given CPU count without a backing semaphore for a shared pool.
+    #[cfg(feature = "testing")]
     pub fn dummy(count: u32) -> Self {
         Self {
             num_cpus: count,

diff --git a/lib/common/issues/src/issue.rs b/lib/common/issues/src/issue.rs
@@ -49,12 +49,13 @@ impl<I: Issue> From<I> for IssueRecord {
     }
 }
 
-#[allow(dead_code)]
+#[cfg(test)]
 #[derive(Clone)]
 pub(crate) struct DummyIssue {
     pub distinctive: String,
 }
 
+#[cfg(test)]
 impl DummyIssue {
     #[cfg(test)]
     pub fn new(distinctive: impl Into<String>) -> Self {
@@ -64,6 +65,7 @@ impl DummyIssue {
     }
 }
 
+#[cfg(test)]
 impl Issue for DummyIssue {
     fn instance_id(&self) -> &str {
         &self.distinctive

diff --git a/lib/segment/Cargo.toml b/lib/segment/Cargo.toml
@@ -12,13 +12,15 @@ edition = "2021"
 multiling-chinese = ["charabia/chinese"]
 multiling-japanese = ["charabia/japanese"]
 multiling-korean = ["charabia/korean"]
+testing = []
 
 [dev-dependencies]
 criterion = "0.5"
 rmp-serde = "~1.3"
 rand_distr = "0.4.3"
 walkdir = "2.5.0"
 rstest = "0.19.0"
+segment = { path = ".", features = ["testing"] }
 proptest = "1.4.0"
 
 [target.'cfg(not(target_os = "windows"))'.dev-dependencies]

diff --git a/lib/segment/src/index/query_optimization/payload_provider.rs b/lib/segment/src/index/query_optimization/payload_provider.rs
@@ -27,6 +27,7 @@ impl PayloadProvider {
     {
         let payload_storage_guard = self.payload_storage.borrow();
         let payload_ptr_opt = match payload_storage_guard.deref() {
+            #[cfg(feature = "testing")]
             PayloadStorageEnum::InMemoryPayloadStorage(s) => {
                 s.payload_ptr(point_id).map(|x| x.into())
             }

diff --git a/lib/segment/src/json_path/mod.rs b/lib/segment/src/json_path/mod.rs
@@ -63,7 +63,7 @@ pub trait JsonPathInterface: Sized + Clone + FromStr<Err = ()> + Display {
 /// # Panics
 ///
 /// Panics if the string is not a valid path. Thus, this function should only be used in tests.
-#[cfg(test)]
+#[cfg(feature = "testing")]
 pub fn path<P: JsonPathInterface>(p: &str) -> P {
     p.parse().unwrap()
 }
diff --git a/lib/segment/src/json_path/v2.rs b/lib/segment/src/json_path/v2.rs
@@ -483,7 +483,7 @@ impl Anonymize for JsonPathV2 {
 mod tests {
     use super::*;
     fn path(s: &str) -> JsonPathV2 {
-        s.parse().unwrap()
+        super::super::path(s)
     }
 
     #[test]

diff --git a/lib/segment/src/lib.rs b/lib/segment/src/lib.rs
@@ -1,5 +1,6 @@
 pub mod common;
 pub mod entry;
+#[cfg(feature = "testing")]
 pub mod fixtures;
 pub mod id_tracker;
 pub mod index;

diff --git a/lib/segment/src/payload_storage/mod.rs b/lib/segment/src/payload_storage/mod.rs
@@ -1,5 +1,7 @@
 pub mod condition_checker;
+#[cfg(feature = "testing")]
 pub mod in_memory_payload_storage;
+#[cfg(feature = "testing")]
 pub mod in_memory_payload_storage_impl;
 pub mod on_disk_payload_storage;
 mod payload_storage_base;

diff --git a/lib/segment/src/payload_storage/payload_storage_enum.rs b/lib/segment/src/payload_storage/payload_storage_enum.rs
@@ -4,18 +4,21 @@ use serde_json::Value;
 use crate::common::operation_error::OperationResult;
 use crate::common::Flusher;
 use crate::json_path::JsonPath;
+#[cfg(feature = "testing")]
 use crate::payload_storage::in_memory_payload_storage::InMemoryPayloadStorage;
 use crate::payload_storage::on_disk_payload_storage::OnDiskPayloadStorage;
 use crate::payload_storage::simple_payload_storage::SimplePayloadStorage;
 use crate::payload_storage::PayloadStorage;
 use crate::types::Payload;
 
 pub enum PayloadStorageEnum {
+    #[cfg(feature = "testing")]
     InMemoryPayloadStorage(InMemoryPayloadStorage),
     SimplePayloadStorage(SimplePayloadStorage),
     OnDiskPayloadStorage(OnDiskPayloadStorage),
 }
 
+#[cfg(feature = "testing")]
 impl From<InMemoryPayloadStorage> for PayloadStorageEnum {
     fn from(a: InMemoryPayloadStorage) -> Self {
         PayloadStorageEnum::InMemoryPayloadStorage(a)
@@ -40,6 +43,7 @@ impl PayloadStorageEnum {
         F: FnMut(PointOffsetType, &Payload) -> OperationResult<bool>,
     {
         match self {
+            #[cfg(feature = "testing")]
             PayloadStorageEnum::InMemoryPayloadStorage(s) => s.iter(callback),
             PayloadStorageEnum::SimplePayloadStorage(s) => s.iter(callback),
             PayloadStorageEnum::OnDiskPayloadStorage(s) => s.iter(callback),
@@ -50,6 +54,7 @@ impl PayloadStorageEnum {
 impl PayloadStorage for PayloadStorageEnum {
     fn assign(&mut self, point_id: PointOffsetType, payload: &Payload) -> OperationResult<()> {
         match self {
+            #[cfg(feature = "testing")]
             PayloadStorageEnum::InMemoryPayloadStorage(s) => s.assign(point_id, payload),
             PayloadStorageEnum::SimplePayloadStorage(s) => s.assign(point_id, payload),
             PayloadStorageEnum::OnDiskPayloadStorage(s) => s.assign(point_id, payload),
@@ -63,6 +68,7 @@ impl PayloadStorage for PayloadStorageEnum {
         key: &JsonPath,
     ) -> OperationResult<()> {
         match self {
+            #[cfg(feature = "testing")]
             PayloadStorageEnum::InMemoryPayloadStorage(s) => {
                 s.assign_by_key(point_id, payload, key)
             }
@@ -73,6 +79,7 @@ impl PayloadStorage for PayloadStorageEnum {
 
     fn payload(&self, point_id: PointOffsetType) -> OperationResult<Payload> {
         match self {
+            #[cfg(feature = "testing")]
             PayloadStorageEnum::InMemoryPayloadStorage(s) => s.payload(point_id),
             PayloadStorageEnum::SimplePayloadStorage(s) => s.payload(point_id),
             PayloadStorageEnum::OnDiskPayloadStorage(s) => s.payload(point_id),
@@ -81,6 +88,7 @@ impl PayloadStorage for PayloadStorageEnum {
 
     fn delete(&mut self, point_id: PointOffsetType, key: &JsonPath) -> OperationResult<Vec<Value>> {
         match self {
+            #[cfg(feature = "testing")]
             PayloadStorageEnum::InMemoryPayloadStorage(s) => s.delete(point_id, key),
             PayloadStorageEnum::SimplePayloadStorage(s) => s.delete(point_id, key),
             PayloadStorageEnum::OnDiskPayloadStorage(s) => s.delete(point_id, key),
@@ -89,6 +97,7 @@ impl PayloadStorage for PayloadStorageEnum {
 
     fn drop(&mut self, point_id: PointOffsetType) -> OperationResult<Option<Payload>> {
         match self {
+            #[cfg(feature = "testing")]
             PayloadStorageEnum::InMemoryPayloadStorage(s) => s.drop(point_id),
             PayloadStorageEnum::SimplePayloadStorage(s) => s.drop(point_id),
             PayloadStorageEnum::OnDiskPayloadStorage(s) => s.drop(point_id),
@@ -97,6 +106,7 @@ impl PayloadStorage for PayloadStorageEnum {
 
     fn wipe(&mut self) -> OperationResult<()> {
         match self {
+            #[cfg(feature = "testing")]
             PayloadStorageEnum::InMemoryPayloadStorage(s) => s.wipe(),
             PayloadStorageEnum::SimplePayloadStorage(s) => s.wipe(),
             PayloadStorageEnum::OnDiskPayloadStorage(s) => s.wipe(),
@@ -105,6 +115,7 @@ impl PayloadStorage for PayloadStorageEnum {
 
     fn flusher(&self) -> Flusher {
         match self {
+            #[cfg(feature = "testing")]
             PayloadStorageEnum::InMemoryPayloadStorage(s) => s.flusher(),
             PayloadStorageEnum::SimplePayloadStorage(s) => s.flusher(),
             PayloadStorageEnum::OnDiskPayloadStorage(s) => s.flusher(),

diff --git a/lib/segment/src/payload_storage/query_checker.rs b/lib/segment/src/payload_storage/query_checker.rs
@@ -1,3 +1,5 @@
+#![cfg_attr(not(feature = "testing"), allow(unused_imports))]
+
 use std::cell::RefCell;
 use std::collections::HashMap;
 use std::ops::Deref;
@@ -205,12 +207,14 @@ where
 }
 
 /// Only used for testing
+#[cfg(feature = "testing")]
 pub struct SimpleConditionChecker {
     payload_storage: Arc<AtomicRefCell<PayloadStorageEnum>>,
     id_tracker: Arc<AtomicRefCell<IdTrackerSS>>,
     empty_payload: Payload,
 }
 
+#[cfg(feature = "testing")]
 impl SimpleConditionChecker {
     pub fn new(
         payload_storage: Arc<AtomicRefCell<PayloadStorageEnum>>,
@@ -224,6 +228,7 @@ impl SimpleConditionChecker {
     }
 }
 
+#[cfg(feature = "testing")]
 impl ConditionChecker for SimpleConditionChecker {
     fn check(&self, point_id: PointOffsetType, query: &Filter) -> bool {
         let payload_storage_guard = self.payload_storage.borrow();

diff --git a/lib/segment/src/segment.rs b/lib/segment/src/segment.rs
@@ -984,6 +984,7 @@ impl Segment {
 
     /// This function is a simplified version of `search_batch` intended for testing purposes.
     #[allow(clippy::too_many_arguments)]
+    #[cfg(feature = "testing")]
     pub fn search(
         &self,
         vector_name: &str,

diff --git a/lib/segment/tests/integration/batch_search_test.rs b/lib/segment/tests/integration/batch_search_test.rs
@@ -13,6 +13,7 @@ use segment::index::hnsw_index::graph_links::GraphLinksRam;
 use segment::index::hnsw_index::hnsw::HNSWIndex;
 use segment::index::hnsw_index::num_rayon_threads;
 use segment::index::VectorIndex;
+use segment::json_path::path;
 use segment::segment_constructor::build_segment;
 use segment::types::{
     Condition, Distance, FieldCondition, Filter, HnswConfig, Indexes, Payload, PayloadSchemaType,
@@ -21,8 +22,6 @@ use segment::types::{
 use serde_json::json;
 use tempfile::Builder;
 
-use crate::utils::path;
-
 #[test]
 fn test_batch_and_single_request_equivalency() {
     let num_vectors: u64 = 1_000;

diff --git a/lib/segment/tests/integration/byte_storage_hnsw_test.rs b/lib/segment/tests/integration/byte_storage_hnsw_test.rs
@@ -14,6 +14,7 @@ use segment::fixtures::payload_fixtures::{random_dense_byte_vector, random_int_p
 use segment::index::hnsw_index::graph_links::GraphLinksRam;
 use segment::index::hnsw_index::hnsw::HNSWIndex;
 use segment::index::{PayloadIndex, VectorIndex};
+use segment::json_path::path;
 use segment::segment_constructor::build_segment;
 use segment::types::{
     Condition, Distance, FieldCondition, Filter, HnswConfig, Indexes, Payload, Range, SearchParams,
@@ -26,8 +27,6 @@ use segment::vector_storage::VectorStorageEnum;
 use serde_json::json;
 use tempfile::Builder;
 
-use crate::utils::path;
-
 const MAX_EXAMPLE_PAIRS: usize = 4;
 
 enum QueryVariant {

diff --git a/lib/segment/tests/integration/byte_storage_quantization_test.rs b/lib/segment/tests/integration/byte_storage_quantization_test.rs
@@ -16,6 +16,7 @@ use segment::index::hnsw_index::graph_links::GraphLinksRam;
 use segment::index::hnsw_index::hnsw::HNSWIndex;
 use segment::index::hnsw_index::num_rayon_threads;
 use segment::index::{PayloadIndex, VectorIndex};
+use segment::json_path::path;
 use segment::segment_constructor::build_segment;
 use segment::types::{
     BinaryQuantizationConfig, CompressionRatio, Condition, Distance, FieldCondition, Filter,
@@ -31,8 +32,6 @@ use segment::vector_storage::VectorStorageEnum;
 use serde_json::json;
 use tempfile::Builder;
 
-use crate::utils::path;
-
 const MAX_EXAMPLE_PAIRS: usize = 4;
 
 enum QueryVariant {

diff --git a/lib/segment/tests/integration/exact_search_test.rs b/lib/segment/tests/integration/exact_search_test.rs
@@ -13,6 +13,7 @@ use segment::index::hnsw_index::graph_links::GraphLinksRam;
 use segment::index::hnsw_index::hnsw::HNSWIndex;
 use segment::index::hnsw_index::num_rayon_threads;
 use segment::index::{PayloadIndex, VectorIndex};
+use segment::json_path::path;
 use segment::segment_constructor::build_segment;
 use segment::types::{
     Condition, Distance, FieldCondition, Filter, HnswConfig, Indexes, Payload, PayloadSchemaType,
@@ -21,8 +22,6 @@ use segment::types::{
 use serde_json::json;
 use tempfile::Builder;
 
-use crate::utils::path;
-
 #[test]
 fn exact_search_test() {
     let stopped = AtomicBool::new(false);

diff --git a/lib/segment/tests/integration/filtrable_hnsw_test.rs b/lib/segment/tests/integration/filtrable_hnsw_test.rs
@@ -15,6 +15,7 @@ use segment::index::hnsw_index::graph_links::GraphLinksRam;
 use segment::index::hnsw_index::hnsw::HNSWIndex;
 use segment::index::hnsw_index::num_rayon_threads;
 use segment::index::{PayloadIndex, VectorIndex};
+use segment::json_path::path;
 use segment::segment_constructor::build_segment;
 use segment::types::{
     Condition, Distance, FieldCondition, Filter, HnswConfig, Indexes, Payload, PayloadSchemaType,
@@ -26,8 +27,6 @@ use segment::vector_storage::query::reco_query::RecoQuery;
 use serde_json::json;
 use tempfile::Builder;
 
-use crate::utils::path;
-
 const MAX_EXAMPLE_PAIRS: usize = 4;
 
 enum QueryVariant {