reorg code, inversion of control design

marcgreen · Dec 5, 2021 · 10c028d · 10c028d
1 parent d4eb6b6
commit 10c028d
Show file tree

Hide file tree

Showing 5 changed files with 363 additions and 355 deletions.
diff --git a/similar-notes/src/data.ts b/similar-notes/src/data.ts
@@ -95,7 +95,7 @@ export async function filterNotesByNotebookName(
   return filteredNotes;
   }
 
-// code borrowed from joplin link graph plugin
+// (re)introduce batch size option
 export async function getAllNotes(): Promise<Map<string, Note>> {
     var allNotes = []
     var page_num = 1;
@@ -123,6 +123,35 @@ export async function getAllNotes(): Promise<Map<string, Note>> {
     return noteMap;
 }
 
+// async function pageNotes(computation: , withBodies: Boolean): Promise<Map<string, Note>> {
+//     var allNotes = []
+//     var page_num = 1;
+//     do {
+// 	// `parent_id` is the ID of the notebook containing the note.
+// 	var notes = await joplin.data.get(['notes'], {
+// 	    fields: withBodies
+// 		? ['id', 'parent_id', 'title', 'body']
+// 		: ['id', 'parent_id', 'title']
+
+// 	    order_by: 'updated_time',
+// 	    order_dir: 'DESC',
+// 	    limit: 100,
+// 	    page: page_num,
+// 	});
+// 	allNotes.push(...notes.items);
+// 	page_num++;
+//     } while (notes.has_more)
+
+//     const noteMap = new Map();
+//     for (const note of allNotes) {
+// 	noteDict = withBodies
+// 	    ? {id: note.id, title: note.title, parent_id: note.parent_id, body: note.body}
+// 	noteMap.set(note.id, noteDict)
+//     }
+//     return noteMap;
+// }
+
+
 
 // Fetches title of every note
 async function getAllNoteTitles(): Promise<Map<string, Note>> {

diff --git a/similar-notes/src/db.ts b/similar-notes/src/db.ts
@@ -0,0 +1,81 @@
+import joplin from 'api';
+const Sqlite3 = joplin.plugins.require('sqlite3').verbose();
+
+const Log = require('electron-log')
+
+export function openDB(embeddingsDBPath) {
+    let db = new Sqlite3.Database(embeddingsDBPath, (err) => {
+	if (err) {
+	    console.error(err.message);
+	    // TODO what to do for main plugin logic? throw exception? return null?
+	    //return null;
+	    throw err;
+	} else {
+	    Log.log('Connected to embeddings db at ', embeddingsDBPath);
+	}
+    });
+
+    return db;
+}
+
+export function deleteEmbedding(db, noteID) {
+    const stmt = db.prepare("DELETE FROM note_embeddings WHERE note_id = ?");
+    stmt.run(noteID).finalize();
+    //console.log('deleted ' + noteID);
+}
+
+export async function loadEmbeddings(db) {
+    Log.log('loading embeddings');
+    //    let prom = null;
+    let notes = new Map();
+    let stmt = null;
+    db.serialize(function() {
+	db.run("CREATE TABLE IF NOT EXISTS note_embeddings (note_id TEXT PRIMARY KEY, embedding TEXT);");
+	//, timestamp DATETIME DEFAULT CURRENT_TIMESTAMP);");
+
+	//console.log('table exists');
+
+	stmt = db.prepare("SELECT note_id, embedding FROM note_embeddings");
+    });
+
+    // sqlite3 doesn't use await/async, so we make our own
+    const rows: Array<object> = await new Promise((resolve, reject) => {
+	stmt.all(function(err, rows) {
+	    if (err) { reject(err); }
+	    resolve(rows);
+	});
+	stmt.finalize();
+    }); // todo throw error on reject
+
+    // console.log('rows', rows);
+    for (const row of rows) {
+	notes.set(row['note_id'], {id: row['note_id'], embedding: row['embedding'].split(" ").map(x => parseFloat(x))});
+    }
+
+    //prom = new Promise(function (resolve, reject) {resolve(notes)});
+    //    let notes = await prom;
+    //console.log('loading notes', [...notes.entries()]);
+    return notes;
+    //db.close();
+}
+
+export function saveEmbeddings(db, idSlice, embeddings) {
+    //console.info('saving', idSlice, embeddings);
+    db.serialize(async function() {
+	let stmt = db.prepare("INSERT INTO note_embeddings (note_id, embedding) VALUES (?,?) ON CONFLICT(note_id) DO UPDATE SET embedding = excluded.embedding");
+
+	// this promise isn't doing what i want. want to essentially force db commit to happen
+	// bc otherwise model crashes the program before things get written... TODO
+	await new Promise((resolve, reject) => {
+	    for (var i = 0; i < idSlice.length; i++) {
+		//console.log(idSlice[i].toString(), ' and ', embeddings[i].join(" "));
+		stmt.run(idSlice[i].toString(), embeddings[i].join(" "));
+	    }
+
+	    stmt.finalize();
+	    resolve();
+	});
+
+	console.info('to db', stmt, idSlice, embeddings);
+    });
+}