segment inspector + small fixes (qdrant#5576)

* segment inspector + small fixes * clippy
AKJUS · Dec 9, 2024 · f2ab5dc · f2ab5dc
1 parent e668067
commit f2ab5dc
Show file tree

Hide file tree

Showing 4 changed files with 92 additions and 6 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -229,6 +229,14 @@ test = false
 bench = false
 required-features = ["service_debug"]
 
+[[bin]]
+name = "segment_inspector"
+path = "src/segment_inspector.rs"
+test = false
+bench = false
+required-features = ["service_debug"]
+
+
 [workspace]
 members = [
     "lib/api",

diff --git a/src/segment_inspector.rs b/src/segment_inspector.rs
@@ -0,0 +1,75 @@
+use std::path::Path;
+use std::sync::atomic::AtomicBool;
+
+use clap::Parser;
+use segment::entry::entry_point::SegmentEntry;
+use segment::segment_constructor::load_segment;
+use segment::types::PointIdType;
+
+#[derive(Parser, Debug)]
+#[command(version, about)]
+struct Args {
+    /// Path to the segment folder. May be a list
+    #[clap(short, long, num_args=1..)]
+    path: Vec<String>,
+
+    /// Print segment info
+    #[clap(long)]
+    info: bool,
+
+    /// Point ID to inspect
+    #[clap(long)]
+    point_id_int: Option<u64>,
+
+    /// Point ID to inspect (UUID)
+    #[clap(long)]
+    point_id_uuid: Option<String>,
+}
+
+fn main() {
+    let args: Args = Args::parse();
+    for segment_path in args.path {
+        let path = Path::new(&segment_path);
+        if !path.exists() {
+            eprintln!("Path does not exist: {segment_path}");
+            continue;
+        }
+        if !path.is_dir() {
+            eprintln!("Path is not a directory: {segment_path}");
+            continue;
+        }
+
+        // Open segment
+
+        let segment = load_segment(path, &AtomicBool::new(false))
+            .unwrap()
+            .unwrap();
+
+        eprintln!(
+            "path = {:#?}, size-points = {}",
+            path,
+            segment.available_point_count()
+        );
+
+        if args.info {
+            let info = segment.info();
+            eprintln!("info = {info:#?}");
+        }
+
+        if let Some(point_id_int) = args.point_id_int {
+            let point_id = PointIdType::NumId(point_id_int);
+
+            let internal_id = segment.get_internal_id(point_id);
+            if internal_id.is_some() {
+                let version = segment.point_version(point_id);
+                let payload = segment.payload(point_id).unwrap();
+                // let vectors = segment.all_vectors(point_id).unwrap();
+
+                println!("Internal ID: {internal_id:?}");
+                println!("Version: {version:?}");
+                println!("Payload: {payload:?}");
+                // println!("Vectors: {vectors:?}");
+            }
+        }
+    }
+}
diff --git a/src/wal_inspector.rs b/src/wal_inspector.rs
@@ -66,13 +66,12 @@ fn print_collection_wal(wal_path: &Path) {
         Ok(wal) => {
             // print all entries
             let mut count = 0;
-            for (idx, op) in wal.read_all(false) {
+            for (idx, op) in wal.read_all(true) {
                 println!("==========================");
-                println!("Entry: {idx}");
-                println!("Operation: {:?}", op.operation);
-                if let Some(clock_tag) = op.clock_tag {
-                    println!("Clock: {clock_tag:?}");
-                }
+                println!(
+                    "Entry: {idx} Operation: {:?} Clock: {:?}",
+                    op.operation, op.clock_tag
+                );
                 count += 1;
             }
             println!("==========================");

diff --git a/tests/consensus_tests/fixtures.py b/tests/consensus_tests/fixtures.py
@@ -85,6 +85,7 @@ def create_collection(
     write_consistency_factor=1,
     timeout=10,
     sharding_method=None,
+    indexing_threshold=20000,
     headers={},
 ):
     # Create collection in peer_url
@@ -97,6 +98,9 @@ def create_collection(
             "replication_factor": replication_factor,
             "write_consistency_factor": write_consistency_factor,
             "sharding_method": sharding_method,
+            "optimizers_config": {
+                "indexing_threshold": indexing_threshold,
+            },
         },
         headers=headers,
     )