Filtering shouldn't timeout during Export, Delete by query, `Upda…

…te by query`, `Cascade deletion of references`, and `Updating async references`. (typesense#1942)
yrayegan · Sep 14, 2024 · ab8dde0 · ab8dde0
1 parent dd02280
commit ab8dde0
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 12 deletions.
diff --git a/include/collection.h b/include/collection.h
@@ -583,7 +583,8 @@ class Collection {
                                   bool enable_lazy_filter = false,
                                   bool enable_typos_for_alpha_numerical_tokens = true) const;
 
-    Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
+    Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result,
+                                const bool& should_timeout = true) const;
 
     Option<bool> get_reference_filter_ids(const std::string& filter_query,
                                           filter_result_t& filter_result,

diff --git a/include/index.h b/include/index.h
@@ -766,7 +766,8 @@ class Index {
 
     Option<bool> do_filtering_with_lock(filter_node_t* const filter_tree_root,
                                         filter_result_t& filter_result,
-                                        const std::string& collection_name = "") const;
+                                        const std::string& collection_name = "",
+                                        const bool& should_timeout = true) const;
 
     Option<bool> do_reference_filtering_with_lock(filter_node_t* const filter_tree_root,
                                                   filter_result_t& filter_result,

diff --git a/src/collection.cpp b/src/collection.cpp
@@ -115,7 +115,7 @@ Option<bool> Collection::update_async_references_with_lock(const std::string& re
                                                            const uint32_t ref_seq_id, const std::string& field_name) {
     // Update reference helper field of the docs matching the filter.
     filter_result_t filter_result;
-    get_filter_ids(filter, filter_result);
+    get_filter_ids(filter, filter_result, false);
 
     if (filter_result.count == 0) {
         return Option<bool>(true);
@@ -602,7 +602,7 @@ Option<nlohmann::json> Collection::update_matching_filter(const std::string& fil
         delete it;
     } else {
         filter_result_t filter_result;
-        auto filter_ids_op = get_filter_ids(_filter_query, filter_result);
+        auto filter_ids_op = get_filter_ids(_filter_query, filter_result, false);
         if(!filter_ids_op.ok()) {
             return Option<nlohmann::json>(filter_ids_op.code(), filter_ids_op.error());
         }
@@ -3564,7 +3564,8 @@ void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<K
     }
 }
 
-Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_result_t& filter_result) const {
+Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_result_t& filter_result,
+                                        const bool& should_timeout) const {
     std::shared_lock lock(mutex);
 
     const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
@@ -3577,7 +3578,7 @@ Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_
         return filter_op;
     }
 
-    return index->do_filtering_with_lock(filter_tree_root, filter_result, name);
+    return index->do_filtering_with_lock(filter_tree_root, filter_result, name, should_timeout);
 }
 
 Option<bool> Collection::get_related_ids(const std::string& ref_field_name, const uint32_t& seq_id,
@@ -4386,7 +4387,7 @@ void Collection::cascade_remove_docs(const std::string& field_name, const uint32
     auto const ref_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;
 
     filter_result_t filter_result;
-    get_filter_ids(ref_helper_field_name + ":" + std::to_string(ref_seq_id), filter_result);
+    get_filter_ids(ref_helper_field_name + ":" + std::to_string(ref_seq_id), filter_result, false);
 
     if (filter_result.count == 0) {
         return;

diff --git a/src/core_api.cpp b/src/core_api.cpp
@@ -1113,7 +1113,7 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share
             export_state->iter_upper_bound = new rocksdb::Slice(export_state->iter_upper_bound_key);
             export_state->it = collectionManager.get_store()->scan(seq_id_prefix, export_state->iter_upper_bound);
         } else {
-            auto filter_ids_op = collection->get_filter_ids(filter_query, export_state->filter_result);
+            auto filter_ids_op = collection->get_filter_ids(filter_query, export_state->filter_result, false);
 
             if(!filter_ids_op.ok()) {
                 res->set(filter_ids_op.code(), filter_ids_op.error());
@@ -1750,9 +1750,8 @@ bool del_remove_documents(const std::shared_ptr<http_req>& req, const std::share
         // destruction of data is managed by req destructor
         req->data = deletion_state;
 
-        search_stop_us = UINT64_MAX; // Filtering shouldn't timeout during delete operation.
         filter_result_t filter_result;
-        auto filter_ids_op = collection->get_filter_ids(simple_filter_query, filter_result);
+        auto filter_ids_op = collection->get_filter_ids(simple_filter_query, filter_result, false);
 
         if(!filter_ids_op.ok()) {
             res->set(filter_ids_op.code(), filter_ids_op.error());

diff --git a/src/index.cpp b/src/index.cpp
@@ -1801,11 +1801,12 @@ bool Index::field_is_indexed(const std::string& field_name) const {
 
 Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_root,
                                            filter_result_t& filter_result,
-                                           const std::string& collection_name) const {
+                                           const std::string& collection_name,
+                                           const bool& should_timeout) const {
     std::shared_lock lock(mutex);
 
     auto filter_result_iterator = filter_result_iterator_t(collection_name, this, filter_tree_root,
-                                                           search_begin_us, search_stop_us);
+                                                           search_begin_us, should_timeout ? search_stop_us : UINT64_MAX);
     auto filter_init_op = filter_result_iterator.init_status();
     if (!filter_init_op.ok()) {
         return filter_init_op;