Merge pull request #23 from AdityaKotwal100/main

Added Multiprocessing support for writes.
saurabh0719 · May 27, 2021 · 53ebf97 · 53ebf97
2 parents d501b91 + 671feec
commit 53ebf97
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 51 deletions.
diff --git a/README.md b/README.md
@@ -517,8 +517,6 @@ View Elara's [release history](https://github.com/saurabh0719/elara/releases/).
 ## Contributors :
 Original author and maintainer - [Saurabh Pujari](https://github.com/saurabh0719)
 
-<br>
-
 Logo design - [Jonah Eapen](https://mobile.twitter.com/jonaheapen98)
 
 

diff --git a/elara/elara.py b/elara/elara.py
@@ -4,12 +4,13 @@
 
 This source code is licensed under the BSD-style license found in the LICENSE file in the root directory of this source tree.
 """
+import multiprocessing
 import os
 import atexit
 from .elarautil import Util
+from .exceptions import InvalidCacheParams
 from .lru import LRU, Cache_obj
 from .status import Status
-from .exceptions import InvalidCacheParams
 
 
 def is_pos(val):
@@ -18,38 +19,22 @@ def is_pos(val):
 
 class Elara:
 
-    from .strings import setnx, append, getset, mget, mset, msetnx, slen
-    from .lists import (
-        lnew,
-        lpush,
-        lextend,
-        lindex,
-        lrange,
-        lrem,
-        lpop,
-        llen,
-        lappend,
-        lexists,
-        linsert,
-    )
-    from .hashtables import hnew, hadd, haddt, hget, hpop, hkeys, hvals, hexists, hmerge
-    from .shared import (
-        retmem,
-        retdb,
-        retkey,
-        commit,
-        exportdb,
-        exportkeys,
-        exportmem,
-        securedb,
-        updatekey,
-    )
+    from .hashtables import (hadd, haddt, hexists, hget, hkeys, hmerge, hnew,
+                             hpop, hvals)
+    from .lists import (lappend, lexists, lextend, lindex, linsert, llen, lnew,
+                        lpop, lpush, lrange, lrem)
+    from .shared import (commit, exportdb, exportkeys, exportmem, retdb,
+                         retkey, retmem, securedb, updatekey)
+    from .strings import append, getset, mget, mset, msetnx, setnx, slen
 
     def __init__(self, path, commitdb, key_path=None, cache_param=None):
         self.path = os.path.expanduser(path)
         self.commitdb = commitdb
+
+        #self.process = None
+
         atexit.register(self._autocommit)
-
+  
         if cache_param == None:
             self.lru = LRU()
             self.max_age = None
@@ -111,11 +96,20 @@ def _load(self):
             self.db = Util.read_plain_db(self)
             self.lru._load(self.db, self.max_age)
 
-    def _dump(self):
+    def _dump(self): #thread
+        lock = multiprocessing.Lock() 
         if self.key:
-            Util.encrypt_and_store(self)  # Enclose in try-catch
+
+            process = multiprocessing.Process(target=Util.encrypt_and_store,args=(self,lock))
+            process.start()
+            process.join() # Enclose in try-catch
         else:
-            Util.store_plain_db(self)
+
+            process = multiprocessing.Process(target=Util.store_plain_db,args=(self,lock))
+            process.start()
+            process.join()
+
+            #Util.store_plain_db(self)
 
     def _autocommit(self):
         if self.commitdb:

diff --git a/elara/elarautil.py b/elara/elarautil.py
@@ -4,6 +4,7 @@
 
 This source code is licensed under the BSD-style license found in the LICENSE file in the root directory of this source tree.
 """
+import multiprocessing
 import os
 from typing import Dict
 from zlib import crc32
@@ -12,12 +13,8 @@
 import safer
 from cryptography.fernet import Fernet
 
-from .exceptions import (
-    FileAccessError,
-    FileKeyError,
-    LoadChecksumError,
-    LoadIncompatibleDB,
-)
+from .exceptions import (FileAccessError, FileKeyError, LoadChecksumError,
+                         LoadIncompatibleDB)
 
 
 class Util:
@@ -57,17 +54,21 @@ def read_plain_db(obj) -> Dict:
             return curr_db
 
     @staticmethod
-    def store_plain_db(obj):
-        with safer.open(obj.path, "wb") as fctx:
-            try:
-                data = msgpack.packb(obj.db)
-                buffer = b"ELDB"
-                buffer += obj.db_format_version.to_bytes(2, "little")
-                buffer += (crc32(data)).to_bytes(4, "little")
-                buffer += data
+    def store_plain_db(obj,lock):
+        data = msgpack.packb(obj.db)
+        buffer = b"ELDB"
+        buffer += obj.db_format_version.to_bytes(2, "little")
+        buffer += (crc32(data)).to_bytes(4, "little")
+        buffer += data
+        try:
+            lock.acquire()
+            with safer.open(obj.path, "wb") as fctx:
                 fctx.write(buffer)
-            except FileExistsError:
-                raise FileAccessError("File already exists")
+                lock.release()
+                return True
+        except:
+            raise FileAccessError("File already exists")
+
 
     @staticmethod
     def read_and_decrypt(obj):
@@ -97,7 +98,8 @@ def read_and_decrypt(obj):
             return None
 
     @staticmethod
-    def encrypt_and_store(obj):
+    def encrypt_and_store(obj,lock):
+         #pass lock maybe.
         if obj.key:
             fernet = Fernet(obj.key)
             db_snapshot = msgpack.packb(obj.db)
@@ -108,11 +110,15 @@ def encrypt_and_store(obj):
             buffer += crc32(encrypted_data).to_bytes(4, "little")
             buffer += encrypted_data
             try:
+                #lock acquire
+                lock.acquire()
                 with safer.open(obj.path, "wb") as file:
                     file.write(buffer)
+                    #lock release
+                    lock.release()
                     return True
             except FileExistsError:
-                raise FileAccessError("File exists")
+                raise FileAccessError("File already exists")
         else:
             return False
-Original file line number
+Diff line change
@@ Expand Up @@
     ## Contributors :
     Original author and maintainer - [Saurabh Pujari](https://github.com/saurabh0719)
-    <br>
     Logo design - [Jonah Eapen](https://mobile.twitter.com/jonaheapen98)
@@ Expand Down @@