Skip to content

Commit

Permalink
[memory][fs-memory] Optimize fs-memory
Browse files Browse the repository at this point in the history
  • Loading branch information
@NikitaZotov
NikitaZotov committed Apr 15, 2023
1 parent 2d6d9a1 commit 6084090
Show file tree
Hide file tree
Showing 8 changed files with 233 additions and 398 deletions.
510 changes: 161 additions & 349 deletions sc-memory/sc-core/sc-store/sc-fs-memory/sc_dictionary_fs_memory.c
View file

Large diffs are not rendered by default.

6 changes: 4 additions & 2 deletions sc-memory/sc-core/sc-store/sc-fs-memory/sc_dictionary_fs_memory.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ typedef struct _sc_dictionary_fs_memory
sc_char * path;
sc_uint64 max_searchable_string_size;

sc_dictionary * terms_offsets_dictionary;
sc_dictionary * terms_string_offsets_dictionary;
sc_char * terms_offsets_path;

sc_dictionary * string_offsets_link_hashes_dictionary;
Expand Down Expand Up @@ -51,7 +51,9 @@ typedef enum _sc_dictionary_fs_memory_status
SC_FS_MEMORY_READ_ERROR
} sc_dictionary_fs_memory_status;

sc_dictionary_fs_memory_status sc_dictionary_fs_memory_initialize(sc_dictionary_fs_memory ** memory, sc_char const * path);
sc_dictionary_fs_memory_status sc_dictionary_fs_memory_initialize(
sc_dictionary_fs_memory ** memory,
sc_char const * path);

sc_dictionary_fs_memory_status sc_dictionary_fs_memory_shutdown(sc_dictionary_fs_memory * memory);

Expand Down
15 changes: 15 additions & 0 deletions sc-memory/sc-core/sc-store/sc-fs-memory/sc_dictionary_fs_memory_private.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,21 @@ sc_bool _sc_dictionary_fs_memory_string_node_destroy(sc_dictionary_node * node,
return SC_TRUE;
}

sc_char * _sc_dictionary_fs_memory_get_first_term(sc_char const * string)
{
static const sc_char delim[] = " ";

sc_uint32 const size = sc_str_len(string);
sc_char copied_string[size + 1];
sc_mem_cpy(copied_string, string, size);
copied_string[size] = '\0';

sc_char * term = strtok(copied_string, delim);
sc_char * copied_term;
sc_str_cpy(copied_term, term, sc_str_len(term));
return copied_term;
}

sc_list * _sc_dictionary_fs_memory_get_string_terms(sc_char const * string)
{
static const sc_char delim[] = " ";
Expand Down
20 changes: 5 additions & 15 deletions sc-memory/sc-core/sc-store/sc-fs-memory/sc_fs_memory.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,26 +83,17 @@ sc_bool sc_fs_memory_shutdown(sc_segment ** segments, sc_bool save_segments)
return SC_FALSE;
}

sc_bool sc_fs_memory_link_string(
sc_addr_hash const link_hash,
sc_char const * string,
sc_uint32 const string_size)
sc_bool sc_fs_memory_link_string(sc_addr_hash const link_hash, sc_char const * string, sc_uint32 const string_size)
{
return !manager->link_string(manager->fs_memory, link_hash, string, string_size);
}

sc_bool sc_fs_memory_get_string_by_link_hash(
sc_addr_hash const link_hash,
sc_char ** string,
sc_uint32 * string_size)
sc_bool sc_fs_memory_get_string_by_link_hash(sc_addr_hash const link_hash, sc_char ** string, sc_uint32 * string_size)
{
return !manager->get_string_by_link_hash(manager->fs_memory, link_hash, string, (sc_uint64 *)string_size);
}

sc_bool sc_fs_memory_get_link_hashes_by_string(
sc_char const * string,
sc_uint32 const string_size,
sc_list ** links)
sc_bool sc_fs_memory_get_link_hashes_by_string(sc_char const * string, sc_uint32 const string_size, sc_list ** links)
{
return !manager->get_link_hashes_by_string(manager->fs_memory, string, string_size, links);
}
Expand All @@ -113,8 +104,7 @@ sc_bool sc_fs_memory_get_link_hashes_by_substring(
sc_list ** link_hashes,
sc_uint32 max_length_to_search_as_prefix)
{
return !manager->get_link_hashes_by_substring(manager->fs_memory,
substring, substring_size, link_hashes);
return !manager->get_link_hashes_by_substring(manager->fs_memory, substring, substring_size, link_hashes);
}

sc_bool sc_fs_memory_get_strings_by_substring(
Expand All @@ -128,7 +118,7 @@ sc_bool sc_fs_memory_get_strings_by_substring(

sc_bool sc_fs_memory_remove_link_string(sc_addr_hash const link_hash)
{
return SC_TRUE; //!manager->remove_link_string(manager->fs_memory, link_hash);
return SC_TRUE; //! manager->remove_link_string(manager->fs_memory, link_hash);
}

// dictionary read, write and save methods
Expand Down
15 changes: 3 additions & 12 deletions sc-memory/sc-core/sc-store/sc-fs-memory/sc_fs_memory.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,31 +79,22 @@ sc_bool sc_fs_memory_shutdown(sc_segment ** segments, sc_bool save_segments);
* @param string A key string
* @param string_size A key string size
*/
sc_bool sc_fs_memory_link_string(
sc_addr_hash link_hash,
sc_char const * string,
sc_uint32 string_size);
sc_bool sc_fs_memory_link_string(sc_addr_hash link_hash, sc_char const * string, sc_uint32 string_size);

/*! Gets sc-link content string with its size.
* @param link_hash A sc-link hash
* @param[out] string A content string
* @param[out] string_size A content string size
*/
sc_bool sc_fs_memory_get_string_by_link_hash(
sc_addr_hash link_hash,
sc_char ** string,
sc_uint32 * string_size);
sc_bool sc_fs_memory_get_string_by_link_hash(sc_addr_hash link_hash, sc_char ** string, sc_uint32 * string_size);

/*! Gets sc-link hashes from file system memory by it string content.
* @param string A key string
* @param string_size A key string size
* @param[out] link_hashes A pointer to sc-link hashes
* @returns SC_TRUE, if sc-link_hashes exist.
*/
sc_bool sc_fs_memory_get_link_hashes_by_string(
sc_char const * string,
sc_uint32 string_size,
sc_list ** link_hashes);
sc_bool sc_fs_memory_get_link_hashes_by_string(sc_char const * string, sc_uint32 string_size, sc_list ** link_hashes);

/*! Gets sc-link hashes from file system memory by it substring content.
* @param substring A key substring
Expand Down
6 changes: 4 additions & 2 deletions sc-memory/sc-core/sc-store/sc_storage.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1112,7 +1112,8 @@ sc_result sc_storage_find_links_by_content_substring(
if (string == null_ptr)
sc_string_empty(string);

sc_result result = sc_fs_memory_get_link_hashes_by_substring(string, string_size, result_hashes, max_length_to_search_as_prefix);
sc_result result =
sc_fs_memory_get_link_hashes_by_substring(string, string_size, result_hashes, max_length_to_search_as_prefix);
sc_mem_free(string);

if (result != SC_RESULT_OK)
Expand Down Expand Up @@ -1140,7 +1141,8 @@ sc_result sc_storage_find_links_contents_by_content_substring(
if (string == null_ptr)
sc_string_empty(string);

sc_result result = sc_fs_memory_get_strings_by_substring(string, string_size, result_strings, max_length_to_search_as_prefix);
sc_result result =
sc_fs_memory_get_strings_by_substring(string, string_size, result_strings, max_length_to_search_as_prefix);
sc_mem_free(string);
if (result != SC_RESULT_OK)
return SC_RESULT_ERROR;
Expand Down
4 changes: 2 additions & 2 deletions sc-memory/sc-memory/sc_scs_helper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,8 +256,8 @@ class StructGenerator
{
if (el.IsURL())
{
ScLink link(m_ctx, linkAddr);
link.Set(el.GetValue());
ScLink link(m_ctx, linkAddr);
link.Set(el.GetValue());
}
else
{
Expand Down
55 changes: 39 additions & 16 deletions sc-memory/tests/sc-memory/fs-storage/sc-fs-storage-api.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_string_by_link_hash_n
{
sc_char * found_string;
sc_uint64 size;
EXPECT_EQ(sc_dictionary_fs_memory_get_string_by_link_hash(memory, hash, &found_string, &size), SC_FS_MEMORY_NO_STRING);
EXPECT_EQ(
sc_dictionary_fs_memory_get_string_by_link_hash(memory, hash, &found_string, &size), SC_FS_MEMORY_NO_STRING);
EXPECT_EQ(found_string, nullptr);
sc_mem_free(found_string);
}
Expand Down Expand Up @@ -262,11 +263,13 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_string_by_link_hash_r
{
sc_char * found_string;
sc_uint64 size;
EXPECT_EQ(sc_dictionary_fs_memory_get_string_by_link_hash(memory, hash1, &found_string, &size), SC_FS_MEMORY_NO_STRING);
EXPECT_EQ(
sc_dictionary_fs_memory_get_string_by_link_hash(memory, hash1, &found_string, &size), SC_FS_MEMORY_NO_STRING);
EXPECT_EQ(found_string, nullptr);
sc_mem_free(found_string);

EXPECT_EQ(sc_dictionary_fs_memory_get_string_by_link_hash(memory, hash2, &found_string, &size), SC_FS_MEMORY_NO_STRING);
EXPECT_EQ(
sc_dictionary_fs_memory_get_string_by_link_hash(memory, hash2, &found_string, &size), SC_FS_MEMORY_NO_STRING);
EXPECT_EQ(found_string, nullptr);
sc_mem_free(found_string);
}
Expand All @@ -290,7 +293,9 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_string
EXPECT_EQ(sc_dictionary_fs_memory_link_string(memory, hash2, string2, sc_str_len(string2)), SC_FS_MEMORY_OK);

sc_list * found_link_hashes;
EXPECT_EQ(sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string1, sc_str_len(string1), &found_link_hashes), SC_FS_MEMORY_OK);
EXPECT_EQ(
sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string1, sc_str_len(string1), &found_link_hashes),
SC_FS_MEMORY_OK);
EXPECT_EQ(found_link_hashes->size, 1u);

sc_iterator * it = sc_list_iterator(found_link_hashes);
Expand All @@ -299,7 +304,9 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_string
sc_iterator_destroy(it);
sc_list_destroy(found_link_hashes);

EXPECT_EQ(sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string2, sc_str_len(string2), &found_link_hashes), SC_FS_MEMORY_OK);
EXPECT_EQ(
sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string2, sc_str_len(string2), &found_link_hashes),
SC_FS_MEMORY_OK);
EXPECT_EQ(found_link_hashes->size, 1u);

it = sc_list_iterator(found_link_hashes);
Expand Down Expand Up @@ -339,7 +346,9 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_string
EXPECT_EQ(sc_dictionary_fs_memory_link_string(memory, hash, string2, sc_str_len(string2)), SC_FS_MEMORY_OK);

sc_list * found_link_hashes;
EXPECT_EQ(sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string2, sc_str_len(string2), &found_link_hashes), SC_FS_MEMORY_OK);
EXPECT_EQ(
sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string2, sc_str_len(string2), &found_link_hashes),
SC_FS_MEMORY_OK);
EXPECT_EQ(found_link_hashes->size, 1u);

sc_iterator * it = sc_list_iterator(found_link_hashes);
Expand All @@ -349,8 +358,6 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_string
sc_list_destroy(found_link_hashes);
}
{
sc_list * strings;
sc_list_init(&strings);
sc_char string1[] = TEXT_EXAMPLE_1;

sc_list * found_link_hashes;
Expand Down Expand Up @@ -383,7 +390,9 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_string

{
sc_list * found_link_hashes;
EXPECT_EQ(sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string1, sc_str_len(string1), &found_link_hashes), SC_FS_MEMORY_OK);
EXPECT_EQ(
sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string1, sc_str_len(string1), &found_link_hashes),
SC_FS_MEMORY_OK);
EXPECT_EQ(found_link_hashes->size, 1u);

sc_iterator * it = sc_list_iterator(found_link_hashes);
Expand All @@ -392,7 +401,9 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_string
sc_iterator_destroy(it);
sc_list_destroy(found_link_hashes);

EXPECT_EQ(sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string2, sc_str_len(string2), &found_link_hashes), SC_FS_MEMORY_OK);
EXPECT_EQ(
sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string2, sc_str_len(string2), &found_link_hashes),
SC_FS_MEMORY_OK);
EXPECT_EQ(found_link_hashes->size, 1u);

it = sc_list_iterator(found_link_hashes);
Expand All @@ -410,7 +421,9 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_string

{
sc_list * found_link_hashes;
EXPECT_EQ(sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string1, sc_str_len(string1), &found_link_hashes), SC_FS_MEMORY_OK);
EXPECT_EQ(
sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string1, sc_str_len(string1), &found_link_hashes),
SC_FS_MEMORY_OK);
EXPECT_EQ(found_link_hashes->size, 1u);

sc_iterator * it = sc_list_iterator(found_link_hashes);
Expand All @@ -419,7 +432,9 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_string
sc_iterator_destroy(it);
sc_list_destroy(found_link_hashes);

EXPECT_EQ(sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string2, sc_str_len(string2), &found_link_hashes), SC_FS_MEMORY_OK);
EXPECT_EQ(
sc_dictionary_fs_memory_get_link_hashes_by_string(memory, string2, sc_str_len(string2), &found_link_hashes),
SC_FS_MEMORY_OK);
EXPECT_EQ(found_link_hashes->size, 1u);

it = sc_list_iterator(found_link_hashes);
Expand All @@ -433,7 +448,6 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_string
EXPECT_EQ(sc_dictionary_fs_memory_shutdown(memory), SC_FS_MEMORY_OK);
}


TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_substring)
{
sc_dictionary_fs_memory * memory;
Expand All @@ -450,7 +464,10 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_substr

sc_list * found_link_hashes;
sc_char substring1[] = "it";
EXPECT_EQ(sc_dictionary_fs_memory_get_link_hashes_by_substring(memory, substring1, sc_str_len(substring1), &found_link_hashes), SC_FS_MEMORY_OK);
EXPECT_EQ(
sc_dictionary_fs_memory_get_link_hashes_by_substring(
memory, substring1, sc_str_len(substring1), &found_link_hashes),
SC_FS_MEMORY_OK);
EXPECT_EQ(found_link_hashes->size, 2u);

sc_iterator * it = sc_list_iterator(found_link_hashes);
Expand All @@ -462,7 +479,10 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_substr
sc_list_destroy(found_link_hashes);

sc_char substring2[] = "it is the first";
EXPECT_EQ(sc_dictionary_fs_memory_get_link_hashes_by_substring(memory, substring2, sc_str_len(substring2), &found_link_hashes), SC_FS_MEMORY_OK);
EXPECT_EQ(
sc_dictionary_fs_memory_get_link_hashes_by_substring(
memory, substring2, sc_str_len(substring2), &found_link_hashes),
SC_FS_MEMORY_OK);
EXPECT_EQ(found_link_hashes->size, 1u);

it = sc_list_iterator(found_link_hashes);
Expand All @@ -472,7 +492,10 @@ TEST(ScDictionaryFsMemoryTest, sc_dictionary_fs_memory_get_link_hashes_by_substr
sc_list_destroy(found_link_hashes);

sc_char substring3[] = "it is the second";
EXPECT_EQ(sc_dictionary_fs_memory_get_link_hashes_by_substring(memory, substring3, sc_str_len(substring3), &found_link_hashes), SC_FS_MEMORY_OK);
EXPECT_EQ(
sc_dictionary_fs_memory_get_link_hashes_by_substring(
memory, substring3, sc_str_len(substring3), &found_link_hashes),
SC_FS_MEMORY_OK);
EXPECT_EQ(found_link_hashes->size, 1u);

it = sc_list_iterator(found_link_hashes);
Expand Down

0 comments on commit 6084090

Please sign in to comment.