Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[add-snapshot] Add snapshot mechanism to enhanced PathOram #1

Draft
wants to merge 5 commits into
base: nachiket-v2
Choose a base branch
from
Draft

[add-snapshot] Add snapshot mechanism to enhanced PathOram #1

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4e9a7cd
Add snapshot feature and tests; modify ORAM to work flexibly
nachiketsrao Nov 22, 2024
ef887d5
New files to enable generating and testing snapshot
nachiketsrao Nov 22, 2024
08c368e
Read tracefile
nachiketsrao Nov 27, 2024
98cce56
Update snapshot generation and testing to use tracefile
nachiketsrao Dec 1, 2024
985a85a
Delete pathOram/snapshot_redis.rdbb
nachiketsrao Dec 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Read tracefile
  • Loading branch information
@nachiketsrao
nachiketsrao committed Nov 27, 2024
commit 08c368eac872e83db6265c32c99bb804c85b4704
19 changes: 19 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,22 @@ To run local Redis server: ./start_redis_serve.sh
To run the tests: ./run_test_suite.sh

To run main: ./makerun.sh


## How to do snapshot:
In generate_db_snapshot.go, change

const (
logCapacity = 10 // Logarithm base 2 of capacity (1024 buckets)
Z = 5 // Number of blocks per bucket
stashSize = 20 // Maximum number of blocks in stash
)

Make sure to place the tracefile and define the path at `TODO: define tracefile path here`

The generated snapshot in : dump.rdb and proxy_snapshot.json in the root of this directory

## How to use snapshot:
Set correct value for arguments logCapacity, Z, stashSize, set the -snapshot flag, tracefile path.

- Make sure that the generate rdb and executor oram code use the same Key with input `oblisqloram`
81 changes: 57 additions & 24 deletions pathOram/tests/generate_db_snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package main

import (
"bufio"
"crypto/sha256"
"encoding/json"
"fmt"
"log"
"os"
"strconv"
"strings"

"pathOram/pkg/oram/oram"
"pathOram/pkg/oram/request"
Expand All @@ -30,7 +31,7 @@ const (

// This function simulates operations on ORAM and stores snapshots of internal data.
func main() {
key_input := "ura"
key_input := "oblisqloram"
// Generate the SHA-256 hash of the input string
hash := sha256.New()
hash.Write([]byte(key_input))
Expand All @@ -47,37 +48,69 @@ func main() {
}
defer o.RedisClient.Close()

totalOperations := 1000 // Number of operations you plan to perform
// totalOperations := 1000 // Number of operations you plan to perform

// Use for testing:
// Create PUT requests
putRequests := make([]request.Request, totalOperations)
for i := 0; i < totalOperations; i++ {
key := strconv.Itoa(i)
putRequests[i] = request.Request{
Key: key,
Value: fmt.Sprintf("Value%d", i),
// putRequests := make([]request.Request, totalOperations)
// for i := 0; i < totalOperations; i++ {
// key := strconv.Itoa(i)
// putRequests[i] = request.Request{
// Key: key,
// Value: fmt.Sprintf("Value%d", i),
// }
// }

// Load data from tracefile and create Request objects
var requests []request.Request
file, err := os.Open(tracefile) // TODO: define tracefile path here
if err != nil {
log.Fatalf("failed to open tracefile: %v", err)
}
defer file.Close()

const maxBufferSize = 1024 * 1024 // 1MB

scanner := bufio.NewScanner(file)
buffer := make([]byte, maxBufferSize)
scanner.Buffer(buffer, maxBufferSize)

for scanner.Scan() {
line := scanner.Text()
// Only process lines that start with "SET"
if strings.HasPrefix(line, "SET") {
parts := strings.SplitN(line, " ", 3)
if len(parts) != 3 {
continue // skip lines that don't have exactly 3 parts
}
key := parts[1]
value := parts[2]
requests = append(requests, request.Request{Key: key, Value: value})
}
}

// Batch size for processing requests
batchSize := 50 // Adjust batch size as needed
if err := scanner.Err(); err != nil {
log.Fatalf("error reading tracefile: %v", err)
}

// Process PUT requests in batches with a progress bar
writeProgress := progressbar.Default(int64(totalOperations), "Writing: ")
// Initialize DB with tracefile contents and display a progress bar
batchSize := 10
bar := progressbar.Default(int64(len(requests)), "Setting values...")

for i := 0; i < totalOperations; i += batchSize {
end := i + batchSize
if end > totalOperations {
end = totalOperations
}
_, err := o.Batching(putRequests[i:end], batchSize)
if err != nil {
log.Fatalf("Error during PUT batching: %v", err)
for start := 0; start < len(requests); start += batchSize {
end := start + batchSize
if end > len(requests) {
end = len(requests) // Ensure we don't go out of bounds
}
writeProgress.Add(end - i)

o.Batching(requests[start:end], batchSize)

// Increment the progress bar by the batch size or remaining items
_ = bar.Add(end - start)
}

writeProgress.Finish()
bar.Finish()
fmt.Println("Finished Initializing DB!")

// Optionally print the ORAM's internal tree (this might be useful for debugging)
// utils.PrintTree(o)
Expand All @@ -98,7 +131,7 @@ func main() {
}

// Open or create the snapshot file
file, err := os.Create("proxy_snapshot.json")
file, err = os.Create("proxy_snapshot.json")
if err != nil {
log.Fatalf("Error creating file: %v", err)
}
Expand Down