Sync engine (silverbulletmd#298)

Fixes silverbulletmd#261
jimt · Jan 13, 2023 · a56e14b · a56e14b
1 parent de6f531
commit a56e14b
Show file tree

Hide file tree

Showing 51 changed files with 1,024 additions and 1,409 deletions.
diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml
@@ -3,7 +3,7 @@ name: Build & Release
 on:
   push:
     tags:
-    - '*'
+      - "*"
 jobs:
   build:
     name: Build (${{ matrix.os }} - ${{ matrix.arch }})
@@ -72,6 +72,7 @@ jobs:
           files: |
             desktop/out/**/*.deb
             desktop/out/**/*Setup.exe
+            desktop/out/**/RELEASES
             desktop/out/**/*.rpm
             desktop/out/**/*.zip
             dist/silverbullet.js
diff --git a/build_mobile.ts b/build_mobile.ts
@@ -1,4 +1,4 @@
-import { bundle, esbuild } from "./build.ts";
+import { bundle, esbuild } from "./build_web.ts";
 import * as flags from "https://deno.land/std@0.165.0/flags/mod.ts";
 import { copy } from "https://deno.land/std@0.165.0/fs/copy.ts";
 

diff --git a/build.ts → build_web.ts b/build.ts → build_web.ts
diff --git a/cmd/server.ts b/cmd/server.ts
@@ -7,6 +7,7 @@ export function serveCommand(options: any, folder: string) {
   const pagesPath = path.resolve(Deno.cwd(), folder);
   const hostname = options.hostname || "127.0.0.1";
   const port = options.port || 3000;
+  const bareMode = options.bare;
 
   console.log(
     "Going to start Silver Bullet binding to",
@@ -27,6 +28,7 @@ export function serveCommand(options: any, folder: string) {
     dbPath: path.join(pagesPath, options.db),
     assetBundle: new AssetBundle(assetBundle as AssetJson),
     user: options.user,
+    bareMode,
   });
   httpServer.start().catch((e) => {
     console.error("HTTP Server error", e);

diff --git a/server/hooks/page_namespace.ts → common/hooks/page_namespace.ts b/server/hooks/page_namespace.ts → common/hooks/page_namespace.ts
diff --git a/common/manifest.ts b/common/manifest.ts
@@ -4,7 +4,7 @@ import { CronHookT } from "../plugos/hooks/cron.deno.ts";
 import { EventHookT } from "../plugos/hooks/event.ts";
 import { CommandHookT } from "../web/hooks/command.ts";
 import { SlashCommandHookT } from "../web/hooks/slash_command.ts";
-import { PageNamespaceHookT } from "../server/hooks/page_namespace.ts";
+import { PageNamespaceHookT } from "./hooks/page_namespace.ts";
 import { CodeWidgetT } from "../web/hooks/code_widget.ts";
 
 export type SilverBulletHooks =

diff --git a/common/spaces/asset_bundle_space_primitives.ts b/common/spaces/asset_bundle_space_primitives.ts
@@ -12,15 +12,15 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
   }
 
   async fetchFileList(): Promise<FileMeta[]> {
-    const l = await this.wrapped.fetchFileList();
+    const files = await this.wrapped.fetchFileList();
     return this.assetBundle.listFiles().filter((p) => p.startsWith("_plug/"))
       .map((p) => ({
         name: p,
         contentType: "application/json",
         lastModified: bootTime,
         perm: "ro",
         size: -1,
-      } as FileMeta)).concat(l);
+      } as FileMeta)).concat(files);
   }
 
   readFile(
@@ -31,7 +31,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
       const data = this.assetBundle.readFileSync(name);
       // console.log("Requested encoding", encoding);
       return Promise.resolve({
-        data: encoding === "string" ? new TextDecoder().decode(data) : data,
+        data: encoding === "utf8" ? new TextDecoder().decode(data) : data,
         meta: {
           lastModified: bootTime,
           size: data.byteLength,
@@ -60,7 +60,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
     name: string,
     encoding: FileEncoding,
     data: FileData,
-    selfUpdate?: boolean | undefined,
+    selfUpdate?: boolean,
   ): Promise<FileMeta> {
     return this.wrapped.writeFile(name, encoding, data, selfUpdate);
   }

diff --git a/common/spaces/disk_space_primitives.ts b/common/spaces/disk_space_primitives.ts
@@ -15,6 +15,8 @@ function lookupContentType(path: string): string {
   return mime.getType(path) || "application/octet-stream";
 }
 
+const excludedFiles = ["data.db", "data.db-journal", "sync.json"];
+
 export class DiskSpacePrimitives implements SpacePrimitives {
   rootPath: string;
 
@@ -48,7 +50,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
       let data: FileData | null = null;
       const contentType = lookupContentType(name);
       switch (encoding) {
-        case "string":
+        case "utf8":
           data = await Deno.readTextFile(localPath);
           break;
         case "dataurl":
@@ -98,7 +100,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
 
       // Actually write the file
       switch (encoding) {
-        case "string":
+        case "utf8":
           await Deno.writeTextFile(`${localPath}`, data as string);
           break;
         case "dataurl":
@@ -165,8 +167,12 @@ export class DiskSpacePrimitives implements SpacePrimitives {
       const fullPath = file.path;
       try {
         const s = await Deno.stat(fullPath);
+        const name = fullPath.substring(this.rootPath.length + 1);
+        if (excludedFiles.includes(name)) {
+          continue;
+        }
         allFiles.push({
-          name: fullPath.substring(this.rootPath.length + 1),
+          name: name,
           lastModified: s.mtime!.getTime(),
           contentType: mime.getType(fullPath) || "application/octet-stream",
           size: s.size,

diff --git a/common/spaces/evented_space_primitives.ts b/common/spaces/evented_space_primitives.ts
@@ -35,7 +35,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
     name: string,
     encoding: FileEncoding,
     data: FileData,
-    selfUpdate: boolean,
+    selfUpdate?: boolean,
   ): Promise<FileMeta> {
     const newMeta = await this.wrapped.writeFile(
       name,
@@ -48,7 +48,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
       const pageName = name.substring(0, name.length - 3);
       let text = "";
       switch (encoding) {
-        case "string":
+        case "utf8":
           text = data as string;
           break;
         case "arraybuffer":

diff --git a/common/spaces/file_meta_space_primitives.ts b/common/spaces/file_meta_space_primitives.ts
@@ -12,10 +12,10 @@ export class FileMetaSpacePrimitives implements SpacePrimitives {
   }
 
   async fetchFileList(): Promise<FileMeta[]> {
-    const list = await this.wrapped.fetchFileList();
+    const files = await this.wrapped.fetchFileList();
     // Enrich the file list with custom meta data (for pages)
     const allFilesMap: Map<string, any> = new Map(
-      list.map((fm) => [fm.name, fm]),
+      files.map((fm) => [fm.name, fm]),
     );
     for (
       const { page, value } of await this.indexSyscalls["index.queryPrefix"](
@@ -53,7 +53,7 @@ export class FileMetaSpacePrimitives implements SpacePrimitives {
     name: string,
     encoding: FileEncoding,
     data: FileData,
-    selfUpdate?: boolean | undefined,
+    selfUpdate?: boolean,
   ): Promise<FileMeta> {
     return this.wrapped.writeFile(name, encoding, data, selfUpdate);
   }

diff --git a/common/spaces/http_space_primitives.ts b/common/spaces/http_space_primitives.ts
@@ -3,33 +3,51 @@ import { Plug } from "../../plugos/plug.ts";
 import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
 import {
   base64DecodeDataUrl,
+  base64Encode,
   base64EncodedDataUrl,
 } from "../../plugos/asset_bundle/base64.ts";
 import { mime } from "../../plugos/deps.ts";
 
 export class HttpSpacePrimitives implements SpacePrimitives {
-  fsUrl: string;
+  private fsUrl: string;
   private plugUrl: string;
 
-  constructor(url: string) {
+  constructor(
+    url: string,
+    readonly user?: string,
+    readonly password?: string,
+    readonly base64Put?: boolean,
+  ) {
     this.fsUrl = url + "/fs";
     this.plugUrl = url + "/plug";
   }
 
   private async authenticatedFetch(
     url: string,
-    options: any,
+    options: Record<string, any>,
   ): Promise<Response> {
+    if (this.user && this.password) {
+      // Explicitly set an auth cookie
+      if (!options.headers) {
+        options.headers = {};
+      }
+      options.headers["cookie"] = `auth=${
+        btoa(`${this.user}:${this.password}`)
+      }`;
+    }
     const result = await fetch(url, options);
-    if (result.status === 401) {
+    if (result.status === 401 || result.redirected) {
       // Invalid credentials, reloading the browser should trigger authentication
-      location.reload();
+      if (typeof location !== "undefined") {
+        location.reload();
+      }
+
       throw Error("Unauthorized");
     }
     return result;
   }
 
-  public async fetchFileList(): Promise<FileMeta[]> {
+  async fetchFileList(): Promise<FileMeta[]> {
     const req = await this.authenticatedFetch(this.fsUrl, {
       method: "GET",
     });
@@ -41,9 +59,12 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     name: string,
     encoding: FileEncoding,
   ): Promise<{ data: FileData; meta: FileMeta }> {
-    const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
-      method: "GET",
-    });
+    const res = await this.authenticatedFetch(
+      `${this.fsUrl}/${encodeURI(name)}`,
+      {
+        method: "GET",
+      },
+    );
     if (res.status === 404) {
       throw new Error(`Page not found`);
     }
@@ -52,7 +73,6 @@ export class HttpSpacePrimitives implements SpacePrimitives {
       case "arraybuffer":
         {
           data = await res.arrayBuffer();
-          // data = await abBlob.arrayBuffer();
         }
         break;
       case "dataurl":
@@ -63,7 +83,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
           );
         }
         break;
-      case "string":
+      case "utf8":
         data = await res.text();
         break;
     }
@@ -82,37 +102,56 @@ export class HttpSpacePrimitives implements SpacePrimitives {
 
     switch (encoding) {
       case "arraybuffer":
-      case "string":
+        // actually we want an Uint8Array
+        body = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
+        break;
+      case "utf8":
         body = data;
         break;
       case "dataurl":
         data = base64DecodeDataUrl(data as string);
         break;
     }
-    const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
-      method: "PUT",
-      headers: {
-        "Content-type": "application/octet-stream",
+    const headers: Record<string, string> = {
+      "Content-Type": "application/octet-stream",
+    };
+    if (this.base64Put) {
+      headers["X-Content-Base64"] = "true";
+      headers["Content-Type"] = "text/plain";
+      body = base64Encode(body);
+    }
+
+    const res = await this.authenticatedFetch(
+      `${this.fsUrl}/${encodeURI(name)}`,
+      {
+        method: "PUT",
+        headers,
+        body,
       },
-      body,
-    });
+    );
     const newMeta = this.responseToMeta(name, res);
     return newMeta;
   }
 
   async deleteFile(name: string): Promise<void> {
-    const req = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
-      method: "DELETE",
-    });
+    const req = await this.authenticatedFetch(
+      `${this.fsUrl}/${encodeURI(name)}`,
+      {
+        method: "DELETE",
+      },
+    );
     if (req.status !== 200) {
       throw Error(`Failed to delete file: ${req.statusText}`);
     }
   }
 
   async getFileMeta(name: string): Promise<FileMeta> {
-    const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
-      method: "OPTIONS",
-    });
+    const res = await this.authenticatedFetch(
+      `${this.fsUrl}/${encodeURI(name)}`,
+      {
+        method: "OPTIONS",
+      },
+    );
     if (res.status === 404) {
       throw new Error(`File not found`);
     }

diff --git a/server/hooks/plug_space_primitives.ts → common/spaces/plug_space_primitives.ts b/server/hooks/plug_space_primitives.ts → common/spaces/plug_space_primitives.ts
@@ -5,7 +5,10 @@ import {
   SpacePrimitives,
 } from "../../common/spaces/space_primitives.ts";
 import { FileMeta } from "../../common/types.ts";
-import { NamespaceOperation, PageNamespaceHook } from "./page_namespace.ts";
+import {
+  NamespaceOperation,
+  PageNamespaceHook,
+} from "../hooks/page_namespace.ts";
 import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts";
 
 export class PlugSpacePrimitives implements SpacePrimitives {
@@ -46,8 +49,8 @@ export class PlugSpacePrimitives implements SpacePrimitives {
         }
       }
     }
-    const result = await this.wrapped.fetchFileList();
-    for (const pm of result) {
+    const files = await this.wrapped.fetchFileList();
+    for (const pm of files) {
       allFiles.push(pm);
     }
     return allFiles;