Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Generalize framework #65

Merged
merged 5 commits into from
Nov 2, 2021
Merged

Generalize framework #65

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c8efb84
Update deps and README.md
jsmassa Oct 12, 2021
cc5a5e9
Remove -path from backing protocol
jsmassa Oct 22, 2021
a116323
restructure migrations
jsmassa Oct 25, 2021
95d66d6
Rename and add exist functions
jsmassa Oct 25, 2021
2443cc0
Remove ac from default store
jsmassa Oct 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Rename and add exist functions
  • Loading branch information
@jsmassa
jsmassa committed Oct 25, 2021
commit 95d66d6ed03219270876a9d69526bb5f2f51d837
286 changes: 151 additions & 135 deletions src/konserve/filestore.clj
View file
Open in desktop

Large diffs are not rendered by default.

28 changes: 14 additions & 14 deletions src/konserve/impl/default.cljc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
-serialize -deserialize
PKeyIterable]]
[konserve.impl.storage-layout :refer [-atomic-move -create-store
-copy -create-blob -delete -exists
-copy -create-blob -delete-blob -blob-exists?
-keys -sync-store
-migratable -migrate -handle-foreign-key
-close -get-lock -sync
Expand Down Expand Up @@ -67,8 +67,8 @@
(update-in old-value rkey up-fn)
(up-fn old-value)))
new-store-key (if (:in-place? config)
store-key
(str store-key ".new"))
store-key
(str store-key ".new"))
backup-store-key (str store-key ".backup")
_ (when (:in-place? config) ;; let's back things up before writing then
(trace "backing up to blob: " backup-store-key " for key " key)
Expand Down Expand Up @@ -152,10 +152,10 @@
(let [{:keys [key-vec base]} env
key (first key-vec)
store-key (key->store-key key)
blob-exists? (<?- (-exists backing store-key env))]
blob-exists? (<?- (-blob-exists? backing store-key env))]
(if blob-exists?
(try
(<?- (-delete backing store-key env))
(<?- (-delete-blob backing store-key env))
true
(catch Exception e
(throw (ex-info "Could not delete key."
Expand Down Expand Up @@ -201,7 +201,7 @@
store-key (key->store-key key)
env (assoc env :store-key store-key)
serializer (get serializers default-serializer)
store-key-exists? (<?- (-exists backing store-key env))
store-key-exists? (<?- (-blob-exists? backing store-key env))
migration-key (<?- (-migratable backing key store-key env))]
(if (and (not store-key-exists?) migration-key)
(<?- (-migrate backing migration-key key-vec serializer read-handlers write-handlers env))
Expand Down Expand Up @@ -239,7 +239,7 @@
(if store-key
(cond
(or (ends-with? store-key ".new")
(ends-with? store-key".backup"))
(ends-with? store-key ".backup"))
(recur keys store-keys)

(ends-with? store-key ".ksv")
Expand Down Expand Up @@ -277,12 +277,12 @@
PEDNKeyValueStore
(-exists? [_ key env]
(async+sync
(:sync? env) *default-sync-translation*
(go-try-
(let [store-key (key->store-key key)]
(or (<?- (-exists backing store-key env))
(<?- (-migratable backing key store-key env))
false)))))
(:sync? env) *default-sync-translation*
(go-try-
(let [store-key (key->store-key key)]
(or (<?- (-blob-exists? backing store-key env))
(<?- (-migratable backing key store-key env))
false)))))
(-get-in [this key-vec not-found opts]
(let [{:keys [sync?]} opts]
(async+sync
Expand Down Expand Up @@ -417,7 +417,7 @@
:buffer-size buffer-size
:msg {:type :read-all-keys-error}}))))

(defn new-default-store
(defn connect-default-store
"Create general store in given base of backing store."
[backing
{:keys [default-serializer serializers compressor encryptor
Expand Down
7 changes: 4 additions & 3 deletions src/konserve/impl/storage_layout.cljc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,16 @@
(defprotocol PBackingStore
"Backing store protocol for default implementation of the high-level konserve protocol."
(-create-blob [this store-key env] "Create a blob object to write a metadata and value into.")
(-delete [this store-key env] "Delete a blob object under path.")
(-exists [this store-key env] "Check whether blob exists under path")
(-delete-blob [this store-key env] "Delete a blob object under path.")
(-blob-exists? [this store-key env] "Check whether blob exists under path")
(-migratable [this key store-key env] "Check if blob exists elsewhere and return a migration key or nil.")
(-migrate [this migration-key key-vec serializer read-handlers write-handlers env] "Use the key returned from -migratable to import blob for key-vec.")
(-copy [this from to env] "Copy a blob from one key to another.")
(-atomic-move [this from to env] "Atomically move (rename) a blob.")
(-create-store [this env] "Create the underlying store.")
(-sync-store [this env] "Synchronize the store. This is only needed if your store does not guarantee durability without this synchronisation command, e.g. fsync in the file system.")
(-delete-store [this env] "Delete the underlying store.")
(-store-exists? [this env] "Check if underlying store already exists.")
(-sync-store [this env] "Synchronize the store. This is only needed if your store does not guarantee durability without this synchronisation command, e.g. fsync in the file system.")
(-keys [this env] "List all the keys representing blobs in the store.")
(-handle-foreign-key [this migration-key serializer read-handlers write-handlers env] "Handle keys not recognized by the current konserve version."))

Expand Down
2 changes: 1 addition & 1 deletion test/konserve/cache_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
(doseq [opts [{:sync? false} {:sync? true}]
:let [<!! (if (:sync? opts) identity <!!)]]
(let [_ (fstore/delete-store "/tmp/cache-store")
test-store (<!! (fstore/new-fs-store "/tmp/cache-store" :opts opts))
test-store (<!! (fstore/connect-fs-store "/tmp/cache-store" :opts opts))
store (k/ensure-cache test-store)]

(testing "Test the cache API."
Expand Down
18 changes: 9 additions & 9 deletions test/konserve/filestore_migration_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
[konserve.old-filestore :as old-store]
[clojure.core.async :refer [go <!!]]
[konserve.core :refer [get bget keys]]
[konserve.filestore :refer [new-fs-store delete-store]]))
[konserve.filestore :refer [connect-fs-store delete-store]]))

(deftest old-filestore-v1
(testing "edn migration single call"
Expand All @@ -13,7 +13,7 @@
_ (dotimes [x 10]
(<!! (old-store/-assoc-in store [x] x)))
list-old-store (<!! (old-store/list-keys store))
new-store (<!! (new-fs-store "/tmp/konserve-fs-migration-test-v1-1" :detect-old-file-schema? true))
new-store (<!! (connect-fs-store "/tmp/konserve-fs-migration-test-v1-1" :detect-old-file-schema? true))
list-keys-new (doall (map #(<!! (get new-store %)) (range 0 10)))]
(are [x y] (= x y)
#{[4] [7] [6] [9] [3] [8] [0] [5] [2] [1]} list-old-store
Expand All @@ -24,7 +24,7 @@
_ (dotimes [x 10]
(<!! (old-store/-assoc-in store [x] x)))
list-old-store (<!! (old-store/list-keys store))
new-store (<!! (new-fs-store "/tmp/konserve-fs-migration-test-v1-2" :detect-old-file-schema? true))
new-store (<!! (connect-fs-store "/tmp/konserve-fs-migration-test-v1-2" :detect-old-file-schema? true))
list-keys-new (<!! (keys new-store))]
(println "new keys" list-keys-new)
(are [x y] (= x y)
Expand All @@ -45,7 +45,7 @@
store (<!! (old-store/new-fs-store-v1 "/tmp/konserve-fs-migration-test-v1-3"))
_ (dotimes [x 10]
(<!! (old-store/-bassoc store x (byte-array (range 10)))))
new-store (<!! (new-fs-store "/tmp/konserve-fs-migration-test-v1-3" :detect-old-file-schema? true))
new-store (<!! (connect-fs-store "/tmp/konserve-fs-migration-test-v1-3" :detect-old-file-schema? true))
_ (dotimes [x 10]
(<!! (bget new-store x
(fn [{:keys [input-stream]}]
Expand All @@ -70,7 +70,7 @@
store (<!! (old-store/new-fs-store-v1 "/tmp/konserve-fs-migration-test-v1-4"))
_ (dotimes [x 10]
(<!! (old-store/-bassoc store x (byte-array (range 10)))))
new-store (<!! (new-fs-store "/tmp/konserve-fs-migration-test-v1-4" :detect-old-file-schema? true))
new-store (<!! (connect-fs-store "/tmp/konserve-fs-migration-test-v1-4" :detect-old-file-schema? true))
list-keys (<!! (keys new-store))]
(are [x y] (= x y)
#{{:type :stale-binary,
Expand All @@ -85,7 +85,7 @@
_ (dotimes [x 10]
(<!! (old-store/-assoc-in store [x] x)))
list-old-store (<!! (old-store/list-keys-v2 store))
new-store (<!! (new-fs-store "/tmp/konserve-fs-migration-test-v2-1" :detect-old-file-schema? true))
new-store (<!! (connect-fs-store "/tmp/konserve-fs-migration-test-v2-1" :detect-old-file-schema? true))
list-keys-new (doall (map #(<!! (get new-store %)) (range 0 10)))]
(are [x y] (= x y)
#{{:key 1, :format :edn}
Expand All @@ -107,7 +107,7 @@
_ (dotimes [x 10]
(<!! (old-store/-assoc-in store [x] x)))
list-old-store (<!! (old-store/list-keys-v2 store))
new-store (<!! (new-fs-store "/tmp/konserve-fs-migration-test-v2-2" :detect-old-file-schema? true))
new-store (<!! (connect-fs-store "/tmp/konserve-fs-migration-test-v2-2" :detect-old-file-schema? true))
list-keys-new (<!! (keys new-store))]
(are [x y] (= x y)
#{{:key 1, :format :edn}
Expand Down Expand Up @@ -136,7 +136,7 @@
store (<!! (old-store/new-fs-store "/tmp/konserve-fs-migration-test-v2-3"))
_ (dotimes [x 10]
(<!! (old-store/-bassoc store x (byte-array (range 10)))))
new-store (<!! (new-fs-store "/tmp/konserve-fs-migration-test-v2-3" :detect-old-file-schema? true))
new-store (<!! (connect-fs-store "/tmp/konserve-fs-migration-test-v2-3" :detect-old-file-schema? true))
_ (dotimes [x 10]
(<!! (bget new-store x
(fn [{:keys [_]}]
Expand All @@ -161,7 +161,7 @@
store (<!! (old-store/new-fs-store "/tmp/konserve-fs-migration-test-v2-4"))
_ (dotimes [x 10]
(<!! (old-store/-bassoc store x (byte-array (range 10)))))
new-store (<!! (new-fs-store "/tmp/konserve-fs-migration-test-v2-4" :detect-old-file-schema? true))
new-store (<!! (connect-fs-store "/tmp/konserve-fs-migration-test-v2-4" :detect-old-file-schema? true))
list-keys (<!! (keys new-store))]
(are [x y] (= x y)
#{{:key 0, :type :binary}
Expand Down
12 changes: 6 additions & 6 deletions test/konserve/filestore_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,26 @@
[clojure.core.async :refer [<!! go chan put! close!] :as async]
[konserve.core :refer :all]
[konserve.compliance-test :refer [compliance-test]]
[konserve.filestore :refer [new-fs-store delete-store]]))
[konserve.filestore :refer [connect-fs-store delete-store]]))

(deftest filestore-compliance-test
(let [folder "/tmp/konserve-fs-comp-test"
_ (delete-store folder)
store (<!! (new-fs-store folder))]
store (<!! (connect-fs-store folder))]
(testing "Compliance test with default config."
(compliance-test store))))

(deftest filestore-compliance-test-no-fsync
(let [folder "/tmp/konserve-fs-comp-test"
_ (delete-store folder)
store (new-fs-store folder :opts {:sync? true} :config {:sync-blob? false})]
store (connect-fs-store folder :opts {:sync? true} :config {:sync-blob? false})]
(testing "Compliance test without syncing."
(compliance-test store))))

(deftest filestore-compliance-test-no-file-lock
(let [folder "/tmp/konserve-fs-comp-test"
_ (delete-store folder)
store (<!! (new-fs-store folder :config {:lock-blob? false}))]
store (<!! (connect-fs-store folder :config {:lock-blob? false}))]
(testing "Compliance test without file locking."
(compliance-test store))))

Expand All @@ -32,7 +32,7 @@
(let [folder "/tmp/konserve-fs-test"
_ (spit "/tmp/foo" (range 1 10))
_ (delete-store folder)
store (<!! (new-fs-store folder))]
store (<!! (connect-fs-store folder))]
(testing "Binary"
(testing "ByteArray"
(let [res-ch (chan)]
Expand Down Expand Up @@ -86,6 +86,6 @@
(put! res-ch (slurp input-stream))))))))
(is (= "foo bar" (<!! res-ch))))))
(delete-store folder)
(let [store (<!! (new-fs-store folder))]
(let [store (<!! (connect-fs-store folder))]
(is (= (<!! (keys store))
#{}))))))
4 changes: 2 additions & 2 deletions test/konserve/gc_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
[clojure.test :refer :all]
[konserve.core :as k]
[konserve.memory :refer [new-mem-store]]
[konserve.filestore :refer [new-fs-store delete-store]]
[konserve.filestore :refer [connect-fs-store delete-store]]
[clojure.core.async :refer [<!! <! go chan put! close!] :as async]))

(deftest memory-store-gc-test
Expand All @@ -29,7 +29,7 @@
(testing "Test the GC."
(let [folder "/tmp/konserve-gc"
_ (delete-store folder)
store (<!! (new-fs-store folder))]
store (<!! (connect-fs-store folder))]
(<!! (k/assoc store :foo :bar))
(<!! (k/assoc-in store [:foo] :bar2))
(<!! (k/assoc-in store [:foo2] :bar2))
Expand Down
4 changes: 2 additions & 2 deletions test/konserve/serializers_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
[konserve.core :as k]
[konserve.memory :refer [new-mem-store]]
[konserve.serializers :refer [fressian-serializer]]
[konserve.filestore :refer [new-fs-store delete-store]])
[konserve.filestore :refer [connect-fs-store delete-store]])
(:import [org.fressian.handlers WriteHandler ReadHandler]))

(def custom-tag "java.util.Date")
Expand All @@ -24,7 +24,7 @@
(testing "Test the custom fressian serializers functionality."
(let [folder "/tmp/konserve-fs-serializers-test"
_ (delete-store folder)
store (<!! (new-fs-store folder :serializers {:FressianSerializer (fressian-serializer custom-read-handler custom-write-handler)}))]
store (<!! (connect-fs-store folder :serializers {:FressianSerializer (fressian-serializer custom-read-handler custom-write-handler)}))]
(is (= (<!! (k/get-in store [:foo]))
nil))
(<!! (k/assoc-in store [:foo] (java.util.Date.)))
Expand Down