Make searching optional

sopherapps · Jan 16, 2023 · 76849e6 · 76849e6
1 parent a85eeb0
commit 76849e6
Show file tree

Hide file tree

Showing 9 changed files with 516 additions and 216 deletions.
diff --git a/.gitignore b/.gitignore
@@ -34,4 +34,5 @@ testdb_close/
 db/
 
 # test db files
-*.scdb
+*.scdb
+*.iscdb
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.2.0] - 2023-01-16
+
+### Added
+
+### Changed
+
+- Changed the `scdb.New()` signature, replacing `maxIndexKeyLen` option with `isSearchEnabled`.
+- Permanently set the maximum index key length to 3
+- Changed benchmarks to compare operations when search is enabled to when search is disabled.
+
+### Fixed
+
 ## [0.1.0] - 2023-01-14
 
 ### Added

diff --git a/README.md b/README.md
diff --git a/examples/sample.go b/examples/sample.go
@@ -21,15 +21,14 @@ func main() {
 	var redundantBlocks uint16 = 1
 	var poolCapacity uint64 = 10
 	var compactionInterval uint32 = 1_800
-	var maxIndexKeyLen uint32 = 3
 
 	store, err := scdb.New(
 		"db",
 		&maxKeys,
 		&redundantBlocks,
 		&poolCapacity,
 		&compactionInterval,
-		&maxIndexKeyLen)
+		true)
 	if err != nil {
 		log.Fatalf("error opening store: %s", err)
 	}

diff --git a/scdb/errors/errors.go b/scdb/errors/errors.go
@@ -31,3 +31,18 @@ func (ecs *ErrCollisionSaturation) Error() string {
 func NewErrCollisionSaturation(k []byte) *ErrCollisionSaturation {
 	return &ErrCollisionSaturation{fmt.Sprintf("no free slot for %s", k)}
 }
+
+// ErrNotSupported is the error when there is an attempt to
+// do an unsupported operation
+type ErrNotSupported struct {
+	op string
+}
+
+func (ens *ErrNotSupported) Error() string {
+	return fmt.Sprintf("%s not supported", ens.op)
+}
+
+// NewErrNotSupported creates a new ErrNotSupported
+func NewErrNotSupported(op string) *ErrNotSupported {
+	return &ErrNotSupported{op}
+}
diff --git a/scdb/internal/buffers/pool.go b/scdb/internal/buffers/pool.go
@@ -230,6 +230,13 @@ func (bp *BufferPool) CompactFile(searchIndex *inverted_index.InvertedIndex) err
 	numOfBlocks := int64(header.NumberOfIndexBlocks)
 	blockSize := int64(header.NetBlockSize)
 
+	if searchIndex != nil {
+		err = searchIndex.Clear()
+		if err != nil {
+			return err
+		}
+	}
+
 	for i := int64(0); i < numOfBlocks; i++ {
 		indexBlock, err := bp.readIndexBlock(i, blockSize)
 		if err != nil {
@@ -274,9 +281,11 @@ func (bp *BufferPool) CompactFile(searchIndex *inverted_index.InvertedIndex) err
 					}
 
 					// update search index
-					err = searchIndex.Add(kv.Key, newKvAddr, kv.Expiry)
-					if err != nil {
-						return err
+					if searchIndex != nil {
+						err = searchIndex.Add(kv.Key, newKvAddr, kv.Expiry)
+						if err != nil {
+							return err
+						}
 					}
 
 					// increment the new file offset

diff --git a/scdb/internal/buffers/pool_test.go b/scdb/internal/buffers/pool_test.go
@@ -440,10 +440,12 @@ func TestBufferPool_CompactFile(t *testing.T) {
 	indexFileName := "testdb_pool.iscdb"
 	defer func() {
 		_ = os.Remove(fileName)
+		_ = os.Remove(indexFileName)
 	}()
 
 	// pre-clean up for right results
 	_ = os.Remove(fileName)
+	_ = os.Remove(indexFileName)
 
 	futureTimestamp := uint64(time.Now().Unix() * 2)
 	neverExpires := values.NewKeyValueEntry([]byte("never_expires"), []byte("bar"), 0)
@@ -639,6 +641,9 @@ func TestBufferPool_GetValue(t *testing.T) {
 
 func TestBufferPool_GetManyKeyValues(t *testing.T) {
 	fileName := "testdb_pool.scdb"
+	defer func() {
+		_ = os.Remove(fileName)
+	}()
 
 	t.Run("GetManyKeyValuesReturnsTheKeyValuesForTheGivenAddresses", func(t *testing.T) {
 		defer func() {

diff --git a/scdb/store.go b/scdb/store.go
@@ -72,11 +72,10 @@ type Store struct {
 //     Compaction is important because it reclaims this space and reduces the size
 //     of the database file.
 //
-//   - `maxIndexKeyLen` - default 3:
-//     The maximum number of characters in each key in the search inverted index
-//     The inverted index is used for full-text search of keys to get all key-values
-//     whose keys start with a given byte array.
-func New(path string, maxKeys *uint64, redundantBlocks *uint16, poolCapacity *uint64, compactionInterval *uint32, maxIndexKeyLen *uint32) (*Store, error) {
+//   - `isSearchEnabled` - default false:
+//     Whether the search capability of the store is enabled.
+//     Note that when search is enabled, `set`, `delete`, `clear`, `compact` operations become slower.
+func New(path string, maxKeys *uint64, redundantBlocks *uint16, poolCapacity *uint64, compactionInterval *uint32, isSearchEnabled bool) (*Store, error) {
 	err := os.MkdirAll(path, 0755)
 	if err != nil {
 		return nil, err
@@ -94,9 +93,12 @@ func New(path string, maxKeys *uint64, redundantBlocks *uint16, poolCapacity *ui
 	}
 
 	searchIndexFilePath := filepath.Join(path, defaultSearchIndexFile)
-	searchIndex, err := inverted_index.NewInvertedIndex(searchIndexFilePath, maxIndexKeyLen, maxKeys, redundantBlocks)
-	if err != nil {
-		return nil, err
+	var searchIndex *inverted_index.InvertedIndex
+	if isSearchEnabled {
+		searchIndex, err = inverted_index.NewInvertedIndex(searchIndexFilePath, nil, maxKeys, redundantBlocks)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	interval := 3_600 * time.Second
@@ -169,7 +171,11 @@ func (s *Store) Set(k []byte, v []byte, ttl *uint64) error {
 			}
 
 			// Update the search index
-			return s.searchIndex.Add(k, prevLastOffset, expiry)
+			if s.searchIndex != nil {
+				return s.searchIndex.Add(k, prevLastOffset, expiry)
+			}
+
+			return nil
 		}
 
 	}
@@ -228,6 +234,10 @@ func (s *Store) Get(k []byte) ([]byte, error) {
 //
 // returns a list of pairs of key-value i.e. `buffers.KeyValuePair`
 func (s *Store) Search(term []byte, skip uint64, limit uint64) ([]buffers.KeyValuePair, error) {
+	if s.searchIndex == nil {
+		return nil, errors.NewErrNotSupported("search")
+	}
+
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
@@ -276,9 +286,14 @@ func (s *Store) Delete(k []byte) error {
 		} // else continue looping
 
 	}
+
 	// if it is not found, no error is thrown
 
-	return s.searchIndex.Remove(k)
+	if s.searchIndex != nil {
+		return s.searchIndex.Remove(k)
+	}
+
+	return nil
 }
 
 // Clear removes all data in the store
@@ -291,7 +306,11 @@ func (s *Store) Clear() error {
 		return err
 	}
 
-	return s.searchIndex.Clear()
+	if s.searchIndex != nil {
+		return s.searchIndex.Clear()
+	}
+
+	return nil
 }
 
 // Compact manually removes dangling key-value pairs in the database file
@@ -336,13 +355,15 @@ func (s *Store) Close() error {
 		return err
 	}
 
-	err = s.searchIndex.Close()
-	if err != nil {
-		return err
+	if s.searchIndex != nil {
+		err = s.searchIndex.Close()
+		if err != nil {
+			return err
+		}
+		s.searchIndex = nil
 	}
 
 	s.header = nil
-	s.searchIndex = nil
 
 	return nil
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -34,4 +34,5 @@ testdb_close/ @@
     db/
     # test db files
-    *.scdb
+    *.scdb
+    *.iscdb